docs: add 'Porting Superpowers to a New Harness' guide

An evergreen guide for adding support for a new harness (IDE, CLI, or agent
runner). Teaches the invariants — automatic session-start bootstrap, skill
discovery/invocation, tool mapping, the acceptance test — and points at the
closest reference integration shape (shell-hook, in-process plugin,
instructions-file / declared context file) to copy. Covers discovery, build,
local install, tmux-driven verification, distribution, and PR submission, with a
live reference-integration index and a gotchas appendix.

Two non-negotiable rules: (1) never edit skill bodies; (2) everything ships
through the harness's own install mechanism — never edit the user's config. When
a plugin installer strips undeclared files, declare the bootstrap as a recognized
component (a manifest contextFileName-style context file the installer preserves
and the harness loads every session), generated at install time from the live
SKILL.md + tool mapping. Surfaced-skill-description bootstrap is the softer
fallback.

Hardened against real end-to-end ports (Antigravity CLI): shapes can compose; a
fork doesn't inherit its parent's behavior; a hook system != a usable
session-start event; verify @-includes AND context-file preservation with a
marker; web-search the docs and study existing plugins; reverse-engineer
undocumented harnesses; print/headless modes may hang; workspace-trust gates
stall tmux; declared context files survive plugin install while undeclared files
are stripped; skills-path registration is per-harness.

.codex-plugin/plugin.json

@@ -21,6 +21,7 @@
     "workflow"
   ],
   "skills": "./skills/",
+  "hooks": "./hooks/hooks-codex.json",
   "interface": {
     "displayName": "Superpowers",
     "shortDescription": "Planning, TDD, debugging, and delivery workflows for coding agents",

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "evals"]
+	path = evals
+	url = git@github.com:prime-radiant-inc/superpowers-evals.git

.opencode/INSTALL.md

@@ -45,7 +45,7 @@ Use OpenCode's native `skill` tool:
 
 ```
 use skill tool to list skills
-use skill tool to load superpowers/brainstorming
+use skill tool to load brainstorming
 ```
 
 ## Updating
@@ -98,11 +98,16 @@ Then use the installed package path in `opencode.json`:
 
 ### Tool mapping
 
-When skills reference Claude Code tools:
-- `TodoWrite` → `todowrite`
-- `Task` with subagents → `@mention` syntax
-- `Skill` tool → OpenCode's native `skill` tool
-- File operations → your native tools
+Skills speak in actions ("create a todo", "dispatch a subagent", "read a file"). On OpenCode these resolve to:
+
+- "Create a todo" / "mark complete in todo list" → `todowrite`
+- `Subagent (general-purpose):` template → `task` tool with `subagent_type: "general"` (or `"explore"` for codebase exploration)
+- "Invoke a skill" → OpenCode's native `skill` tool
+- "Read a file" → `read`
+- "Create a file" / "edit a file" / "delete a file" → `apply_patch`
+- "Run a shell command" → `bash`
+- "Search file contents" / "find files by name" → `grep`, `glob`
+- "Fetch a URL" → `webfetch`
 
 ## Getting Help
 

.opencode/plugins/superpowers.js

@@ -1,7 +1,7 @@
 /**
  * Superpowers plugin for OpenCode.ai
  *
- * Injects superpowers bootstrap context via system prompt transform.
+ * Injects superpowers bootstrap context via message transform.
  * Auto-registers skills directory via config hook (no symlinks needed).
  */
 
@@ -74,11 +74,15 @@ export const SuperpowersPlugin = async ({ client, directory }) => {
     const { content } = extractAndStripFrontmatter(fullContent);
 
     const toolMapping = `**Tool Mapping for OpenCode:**
-When skills reference tools you don't have, substitute OpenCode equivalents:
-- \`TodoWrite\` → \`todowrite\`
-- \`Task\` tool with subagents → Use OpenCode's subagent system (@mention)
-- \`Skill\` tool → OpenCode's native \`skill\` tool
-- \`Read\`, \`Write\`, \`Edit\`, \`Bash\` → Your native tools
+When skills request actions, substitute OpenCode equivalents:
+- Create or update todos → \`todowrite\`
+- \`Subagent (general-purpose):\` → \`task\` with \`subagent_type: "general"\`
+- Invoke a skill → OpenCode's native \`skill\` tool
+- Read files → \`read\`
+- Create, edit, or delete files → \`apply_patch\`
+- Run shell commands → \`bash\`
+- Search files → \`grep\`, \`glob\`
+- Fetch a URL → \`webfetch\`
 
 Use OpenCode's native \`skill\` tool to list and load skills.`;
 

.pi/extensions/superpowers.ts

@@ -0,0 +1,121 @@
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+const EXTREMELY_IMPORTANT_MARKER = "<EXTREMELY_IMPORTANT>";
+const BOOTSTRAP_MARKER = "superpowers:using-superpowers bootstrap for pi";
+
+const extensionDir = dirname(fileURLToPath(import.meta.url));
+const packageRoot = resolve(extensionDir, "../..");
+const skillsDir = resolve(packageRoot, "skills");
+const bootstrapSkillPath = resolve(skillsDir, "using-superpowers", "SKILL.md");
+
+let cachedBootstrap: string | null | undefined;
+
+export default function superpowersPiExtension(pi: ExtensionAPI) {
+	let injectBootstrap = true;
+
+	pi.on("resources_discover", async () => ({
+		skillPaths: [skillsDir],
+	}));
+
+	pi.on("session_start", async () => {
+		injectBootstrap = true;
+	});
+
+	pi.on("session_compact", async () => {
+		injectBootstrap = true;
+	});
+
+	pi.on("agent_end", async () => {
+		injectBootstrap = false;
+	});
+
+	pi.on("context", async (event) => {
+		if (!injectBootstrap) return;
+		if (event.messages.some(messageContainsBootstrap)) return;
+
+		const bootstrap = getBootstrapContent();
+		if (!bootstrap) return;
+
+		const bootstrapMessage = {
+			role: "user" as const,
+			content: [{ type: "text" as const, text: bootstrap }],
+			timestamp: Date.now(),
+		};
+
+		const insertAt = firstNonCompactionSummaryIndex(event.messages);
+		return {
+			messages: [
+				...event.messages.slice(0, insertAt),
+				bootstrapMessage,
+				...event.messages.slice(insertAt),
+			],
+		};
+	});
+}
+
+function getBootstrapContent(): string | null {
+	if (cachedBootstrap !== undefined) return cachedBootstrap;
+
+	try {
+		const skillContent = readFileSync(bootstrapSkillPath, "utf8");
+		const body = stripFrontmatter(skillContent);
+		cachedBootstrap = `${EXTREMELY_IMPORTANT_MARKER}
+${BOOTSTRAP_MARKER}
+
+You have superpowers.
+
+The using-superpowers skill content is included below and is already loaded for this Pi session. Follow it now. Do not try to load using-superpowers again.
+
+${body}
+
+${piToolMapping()}
+</EXTREMELY_IMPORTANT>`;
+		return cachedBootstrap;
+	} catch {
+		cachedBootstrap = null;
+		return null;
+	}
+}
+
+function stripFrontmatter(content: string): string {
+	const match = content.match(/^---\n[\s\S]*?\n---\n([\s\S]*)$/);
+	return (match ? match[1] : content).trim();
+}
+
+function piToolMapping(): string {
+	return `## Pi tool mapping
+
+Pi has native skills but does not expose Claude Code's \`Skill\` tool. When a Superpowers instruction says to invoke a skill, use Pi's native skill system instead: load the relevant \`SKILL.md\` with \`read\` when the skill applies, or let a human invoke \`/skill:name\` explicitly.
+
+Pi's built-in coding tools are lowercase: \`read\`, \`write\`, \`edit\`, \`bash\`, plus optional \`grep\`, \`find\`, and \`ls\`. Use those for the corresponding actions: read a file, create or edit files, run shell commands, search file contents, find files by name, and list directories.
+
+Pi does not ship a standard subagent tool. If a subagent tool such as \`subagent\` from \`pi-subagents\` is available, use it for Superpowers subagent workflows. If no subagent tool is available, do the work in this session or explain the missing capability instead of inventing \`Task\` calls.
+
+Pi does not ship a standard task-list tool. If an installed todo/task tool is available, use it. Otherwise track work in plan files or a repo-local \`TODO.md\` when task tracking is needed. Treat older \`TodoWrite\` references as this task-tracking action.`;
+}
+
+function messageContainsBootstrap(message: unknown): boolean {
+	const content = (message as { content?: unknown }).content;
+	if (typeof content === "string") return content.includes(BOOTSTRAP_MARKER);
+	if (!Array.isArray(content)) return false;
+	return content.some((part) => {
+		return (
+			part &&
+			typeof part === "object" &&
+			(part as { type?: unknown }).type === "text" &&
+			typeof (part as { text?: unknown }).text === "string" &&
+			(part as { text: string }).text.includes(BOOTSTRAP_MARKER)
+		);
+	});
+}
+
+function firstNonCompactionSummaryIndex(messages: unknown[]): number {
+	let index = 0;
+	while ((messages[index] as { role?: unknown } | undefined)?.role === "compactionSummary") {
+		index += 1;
+	}
+	return index;
+}

CLAUDE.md

@@ -96,7 +96,7 @@ Skills are not prose — they are code that shapes agent behavior. If you modify
 
 ## Eval harness
 
-Skill-behavior evals live at `evals/` — see `evals/README.md`. Drill (the harness) drives real tmux sessions of Claude Code / Codex / Gemini CLI and judges skill compliance with an LLM verifier. Plugin-infrastructure tests still live at `tests/`.
+Skill-behavior evals live in the `evals/` submodule — after cloning, run `git submodule update --init evals`, then see `evals/README.md`. Drill (the harness) drives real tmux sessions of Claude Code / Codex / Gemini CLI and judges skill compliance with an LLM verifier. Plugin-infrastructure tests still live at `tests/`.
 
 ## Understand the Project Before Contributing
 

README.md

@@ -4,7 +4,7 @@ Superpowers is a complete software development methodology for your coding agent
 
 ## Quickstart
 
-Give your agent Superpowers: [Claude Code](#claude-code), [Codex CLI](#codex-cli), [Codex App](#codex-app), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [OpenCode](#opencode), [Cursor](#cursor), [GitHub Copilot CLI](#github-copilot-cli).
+Give your agent Superpowers: [Claude Code](#claude-code), [Codex App](#codex-app), [Codex CLI](#codex-cli), [Cursor](#cursor), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [GitHub Copilot CLI](#github-copilot-cli), [OpenCode](#opencode), [Pi](#pi).
 
 ## How it works
 
@@ -14,7 +14,7 @@ Once it's teased a spec out of the conversation, it shows it to you in chunks sh
 
 After you've signed off on the design, your agent puts together an implementation plan that's clear enough for an enthusiastic junior engineer with poor taste, no judgement, no project context, and an aversion to testing to follow. It emphasizes true red/green TDD, YAGNI (You Aren't Gonna Need It), and DRY. 
 
-Next up, once you say "go", it launches a *subagent-driven-development* process, having agents work through each engineering task, inspecting and reviewing their work, and continuing forward. It's not uncommon for Claude to be able to work autonomously for a couple hours at a time without deviating from the plan you put together.
+Next up, once you say "go", it launches a *subagent-driven-development* process, having agents work through each engineering task, inspecting and reviewing their work, and continuing forward. It's not uncommon for your agent to work autonomously for a couple hours at a time without deviating from the plan you put together.
 
 There's a bunch more to it, but that's the core of the system. And because the skills trigger automatically, you don't need to do anything special. Your coding agent just has Superpowers.
 
@@ -60,6 +60,14 @@ The Superpowers marketplace provides Superpowers and some other related plugins
   /plugin install superpowers@superpowers-marketplace
   ```
 
+### Codex App
+
+Superpowers is available via the [official Codex plugin marketplace](https://github.com/openai/plugins).
+
+- In the Codex app, click on Plugins in the sidebar.
+- You should see `Superpowers` in the Coding section.
+- Click the `+` next to Superpowers and follow the prompts.
+
 ### Codex CLI
 
 Superpowers is available via the [official Codex plugin marketplace](https://github.com/openai/plugins).
@@ -78,13 +86,15 @@ Superpowers is available via the [official Codex plugin marketplace](https://git
 
 - Select `Install Plugin`.
 
-### Codex App
+### Cursor
 
-Superpowers is available via the [official Codex plugin marketplace](https://github.com/openai/plugins).
+- In Cursor Agent chat, install from marketplace:
 
-- In the Codex app, click on Plugins in the sidebar.
-- You should see `Superpowers` in the Coding section.
-- Click the `+` next to Superpowers and follow the prompts.
+  ```text
+  /add-plugin superpowers
+  ```
+
+- Or search for "superpowers" in the plugin marketplace.
 
 ### Factory Droid
 
@@ -114,29 +124,6 @@ Superpowers is available via the [official Codex plugin marketplace](https://git
   gemini extensions update superpowers
   ```
 
-### OpenCode
-
-OpenCode uses its own plugin install; install Superpowers separately even if you
-already use it in another harness.
-
-- Tell OpenCode:
-
-  ```
-  Fetch and follow instructions from https://raw.githubusercontent.com/obra/superpowers/refs/heads/main/.opencode/INSTALL.md
-  ```
-
-- Detailed docs: [docs/README.opencode.md](docs/README.opencode.md)
-
-### Cursor
-
-- In Cursor Agent chat, install from marketplace:
-
-  ```text
-  /add-plugin superpowers
-  ```
-
-- Or search for "superpowers" in the plugin marketplace.
-
 ### GitHub Copilot CLI
 
 - Register the marketplace:
@@ -151,6 +138,35 @@ already use it in another harness.
   copilot plugin install superpowers@superpowers-marketplace
   ```
 
+### OpenCode
+
+OpenCode uses its own plugin install; install Superpowers separately even if you
+already use it in another harness.
+
+- Tell OpenCode:
+
+  ```
+  Fetch and follow instructions from https://raw.githubusercontent.com/obra/superpowers/refs/heads/main/.opencode/INSTALL.md
+  ```
+
+- Detailed docs: [docs/README.opencode.md](docs/README.opencode.md)
+
+### Pi
+
+Install Superpowers as a Pi package from this repository:
+
+```bash
+pi install git:github.com/obra/superpowers
+```
+
+For local development, run Pi with this checkout loaded as a temporary package:
+
+```bash
+pi -e /path/to/superpowers
+```
+
+The Pi package loads the Superpowers skills and a small extension that injects the `using-superpowers` bootstrap at session startup and again after compaction. Pi has native skills, so no compatibility `Skill` tool is required. Subagent and task-list tools remain optional Pi companion packages.
+
 ## The Basic Workflow
 
 1. **brainstorming** - Activates before writing code. Refines rough ideas through questions, explores alternatives, presents design in sections for validation. Saves design document.
@@ -214,7 +230,7 @@ The general contribution process for Superpowers is below. Keep in mind that we
 4. Follow the `writing-skills` skill for creating and testing new and modified skills
 5. Submit a PR, being sure to fill in the pull request template.
 
-Skill-behavior tests use the eval harness at `evals/`. See `evals/README.md` for setup. Plugin-infrastructure tests live at `tests/` and run via the relevant `run-*.sh` or `npm test`.
+Skill-behavior tests use the eval harness submodule at `evals/`. After cloning this repo, run `git submodule update --init evals`, then see `evals/README.md` for setup. Plugin-infrastructure tests live at `tests/` and run via the relevant `run-*.sh` or `npm test`.
 
 See `skills/writing-skills/SKILL.md` for the complete guide.
 

docs/README.opencode.md

@@ -50,7 +50,7 @@ use skill tool to list skills
 ### Loading a Skill
 
 ```
-use skill tool to load superpowers/brainstorming
+use skill tool to load brainstorming
 ```
 
 ### Personal Skills
@@ -99,17 +99,23 @@ To pin a specific version, use a branch or tag:
 
 The plugin does two things:
 
-1. **Injects bootstrap context** via the `experimental.chat.system.transform` hook, adding superpowers awareness to every conversation.
+1. **Injects bootstrap context** via the `experimental.chat.messages.transform` hook, adding superpowers awareness to every conversation.
 2. **Registers the skills directory** via the `config` hook, so OpenCode discovers all superpowers skills without symlinks or manual config.
 
 ### Tool Mapping
 
-Skills written for Claude Code are automatically adapted for OpenCode:
+Skills speak in actions rather than naming any one runtime's tools. On OpenCode these resolve to:
 
-- `TodoWrite` → `todowrite`
-- `Task` with subagents → OpenCode's `@mention` system
-- `Skill` tool → OpenCode's native `skill` tool
-- File operations → Native OpenCode tools
+- "Create a todo" / "mark complete in todo list" → `todowrite`
+- `Subagent (general-purpose):` template → OpenCode's `task` tool with `subagent_type: "general"` (or `"explore"` for codebase exploration)
+- "Invoke a skill" → OpenCode's native `skill` tool
+- "Read a file" → `read`
+- "Create a file" / "edit a file" / "delete a file" → `apply_patch`
+- "Run a shell command" → `bash`
+- "Search file contents" / "find files by name" → `grep`, `glob`
+- "Fetch a URL" → `webfetch`
+
+(Verified against the installed OpenCode CLI's tool inventory.)
 
 ## Troubleshooting
 
@@ -147,7 +153,7 @@ Then use the installed package path in `opencode.json`:
 
 ### Bootstrap not appearing
 
-1. Check OpenCode version supports `experimental.chat.system.transform` hook
+1. Check OpenCode version supports `experimental.chat.messages.transform` hook
 2. Restart OpenCode after config changes
 
 ## Getting Help

docs/porting-to-a-new-harness.md

@@ -0,0 +1,826 @@
+# Porting Superpowers to a New Harness
+
+This guide explains how to add support for a new harness — an IDE, CLI, or
+agent runner that isn't Claude Code — so that Superpowers skills auto-trigger
+there the same way they do natively.
+
+It is written in two layers. **Part 1–3** explain how the system works and how
+to tell whether a harness can be supported at all; read these before you touch
+anything. **Part 4–8** are a prescriptive procedure for an agent (supervised by
+a human partner) to execute the port end to end, through distribution. An
+appendix indexes the current reference integrations so you can copy the closest
+one.
+
+The integration mechanism differs across harnesses, and it will keep changing.
+This guide deliberately teaches the **invariants** — the things that must be
+true no matter the mechanism — and points you at a live reference implementation
+to copy. When this guide and the code disagree, the code wins; fix the guide.
+
+## Before you start
+
+Adding a harness is the highest-stakes contribution type in this repo. Before
+writing anything:
+
+- Read `CLAUDE.md` and `.github/PULL_REQUEST_TEMPLATE.md` in full — the
+  contributor rules and the new-harness PR requirements are not optional.
+- Search open **and closed** PRs for a prior attempt at this harness. If one
+  exists, understand why it stalled before starting your own.
+
+---
+
+## Part 1 — How Superpowers works across harnesses
+
+Superpowers is the same content everywhere. What changes per harness is the thin
+layer that delivers that content to the model and translates its instructions
+into the harness's native tools. Three components:
+
+1. **Skills (harness-agnostic).** Everything in `skills/` is the source of
+   truth, shared verbatim by every harness. Skills are written to describe
+   *actions* — "invoke a skill", "read a file", "dispatch a subagent", "create a
+   todo" — and never name a specific tool. This is what lets one skill body run
+   on Claude Code, Codex, Gemini, pi, and the rest without edits.
+
+2. **Tool mapping (per-harness).** Each harness needs the action vocabulary
+   translated into its real tool names. That translation lives in
+   `skills/using-superpowers/references/<harness>-tools.md` and/or inline in the
+   harness's bootstrap injector (see Part 5). It says, e.g., "*dispatch a
+   subagent* → call `task` with `subagent_type`."
+
+3. **Bootstrap (per-harness).** At the start of every session, the full
+   `skills/using-superpowers/SKILL.md` is injected into the model's context,
+   wrapped in `<EXTREMELY_IMPORTANT>` tags, with the tool mapping appended. That
+   injected skill is what teaches the model that skills exist and that it must
+   check for a relevant skill before acting. **The bootstrap is the entire
+   integration.** Without it, the skill files are inert — present on disk, never
+   invoked.
+
+### Two rules that make this work
+
+**1. Skills name actions, not tools.** Do **not** edit skill bodies to fit your
+harness. Porting adds a tool-mapping reference and a bootstrap injector; it
+never reaches into `skills/*/SKILL.md` to swap tool names. (The project's
+contributor guidelines treat skill content as carefully-tuned behavior-shaping
+code; rewording it for "compliance" is rejected on sight.)
+
+**2. Everything ships through the harness's own install mechanism. Never edit the
+user's files.** The bootstrap, the skills, and the tool mapping all get delivered
+*as part of what the harness installs* — a plugin, an extension, a marketplace
+entry, an extension-bundled context file. A port **must not** reach into a user's
+global or personal config (`~/.gemini/config/AGENTS.md`, `settings.json`,
+`trustedFolders.json`, a hand-edited `~/.bashrc`, etc.) to inject anything. The
+harness owns what it loads; your install artifact is the only thing you get to
+write. If the install mechanism genuinely can't carry the bootstrap, that is a
+limitation to surface (Part 6) — never a license to hand-edit the user's config.
+(Shape C is *not* an exception: Gemini's context file is fine because it ships
+*inside the installed extension* and is declared by the manifest's
+`contextFileName` — the harness loads the extension's own file, not a file you
+edited in the user's home.)
+
+---
+
+## Part 2 — Can this harness be supported?
+
+A harness can support Superpowers only if it can do all of the following. Check
+these before writing code — if the first one fails, stop.
+
+### Hard requirement: automatic session-start injection
+
+The harness must let you inject text into the model's context **at the start of
+every session, with no per-session opt-in by your human partner.** This is the
+one non-negotiable capability. It can take any form:
+
+- a **hook/event system** that runs a shell command at session start and reads
+  its stdout (Claude Code, Codex, Cursor, Copilot CLI), or
+- an **in-process plugin/extension** with a session-start or message lifecycle
+  callback that can mutate the message array (OpenCode, pi), or
+- an **instructions-file** convention where the harness loads a context file that
+  *your installed extension ships and declares* (e.g. Gemini's `contextFileName`
+  pointing at the extension's own `GEMINI.md`) — not a file you edit in the user's
+  home.
+
+If the only way to get Superpowers in front of the model is for your human
+partner to opt in each session (paste a prompt, run a command, enable a mode),
+the harness
+**cannot** be properly supported. The acceptance test in Part 3 will fail, and
+the PR will be closed. This is the single most common reason a "port" isn't a
+real port.
+
+### The rest of the capability checklist
+
+| Capability | Why it's needed | If absent |
+|---|---|---|
+| **Skill discovery + invocation** | The model must be able to load a skill's full content on demand | If there's no native skill tool, the sanctioned fallback is to `read` the relevant `SKILL.md` directly — see Part 5. A harness with neither a skill tool nor file-read cannot work. |
+| **File read / write / edit** | Nearly every skill manipulates files | Essential. No workaround. |
+| **Run shell commands** | TDD, verification, git workflows | Essential. |
+| **Subagent / task dispatch** | `dispatching-parallel-agents`, `subagent-driven-development` | Degradable: if unavailable, those specific skills tell the model to do the work inline or report the missing capability — *never* to invent a `Task` call. Some harnesses gate this behind a config flag (e.g. Codex needs multi-agent enabled). |
+| **Todo / task tracking** | Progress tracking in several skills | Degradable: fall back to a plan file or `TODO.md`. |
+| **Web fetch / search** | A few skills | Degradable. |
+| **Shell or polyglot script execution (Windows)** | Only for the shell-hook shape, only if you want Windows support | See Part 7. In-process-plugin harnesses sidestep this entirely. |
+
+"Degradable" means: the skill already has fallback wording for the missing
+tool. Your job in the tool mapping is to point at the real tool when it exists
+and reuse that fallback wording when it doesn't.
+
+### You may not need a new directory at all
+
+Some "new harnesses" are really existing integrations under a different
+installer. Factory's Droid, for example, consumes the Claude Code plugin via its
+own `plugin install` command and needs no new files here. Before building,
+check whether the harness can simply load an existing manifest. A port that adds
+nothing to this repo but a paragraph in the README is a perfectly good outcome.
+
+---
+
+## Part 3 — Definition of done
+
+A port is finished when **all** of these are true:
+
+1. The `using-superpowers` bootstrap loads at session start, every session, with
+   no per-session opt-in.
+2. A tool mapping exists for the harness (in
+   `references/<harness>-tools.md`, inline in the bootstrap, or both — per Part 5).
+3. Skills can actually be invoked — natively, or via the documented
+   read-`SKILL.md` fallback — and the model follows them.
+4. **The acceptance test passes.** In a clean session, the user message:
+
+   > Let's make a react todo list
+
+   auto-triggers the `brainstorming` skill *before any code is written*. Capture
+   the full transcript — the PR requires it.
+5. Tests cover the integration (Part 5) and pass.
+6. A real user can install it through the harness's own mechanism (not by
+   hand-copying files), and the version is tracked in `.version-bump.json` where
+   applicable (Part 6). Note that some installers rewrite or strip the manifest on
+   install (one drops it to just `{"name": …}`), so "the *installed* files report
+   the repo version" is not always achievable — track the version at the source
+   manifest and don't treat a rewritten installed manifest as a failure.
+
+A quick smoke check before the full acceptance test: start a session and ask the
+model to describe its superpowers. If the bootstrap injected, it knows it has
+them. (OpenCode's install doc uses `opencode run --print-logs "hello" 2>&1 |
+grep -i superpowers` for the same goal via a different mechanism — log-grep
+rather than asking the model; the `2>&1` matters because logs go to stderr. Find
+your harness's equivalent.)
+
+---
+
+## Part 4 — Choose your integration shape
+
+There are three structural shapes, distinguished by *how you get the bootstrap
+in front of the model*. Pick the one that matches what your harness exposes,
+then copy that reference implementation. The shape determines almost everything
+in Part 5 — the steps below branch on it.
+
+### How to tell which shape you have
+
+Before routing, learn the harness's *actual* mechanism — and don't assume it's
+well documented or that it behaves like whatever harness it forked from.
+
+**Find the surface:**
+
+- **Search the web for the harness's docs** (extension / plugin / hook / skill /
+  MCP / "context file" / "rules file"). Vendor tools change fast; search rather
+  than trust training knowledge.
+- **Find and read an existing third-party extension/plugin for the harness.** A
+  real working example beats docs — it shows the manifest shape, the install
+  command, and which components the harness actually loads.
+- Check what the harness loads at startup: a settings file? an extensions
+  directory? a per-project or global instructions file (`AGENTS.md`, `<NAME>.md`)?
+
+**If it's underdocumented, reverse-engineer it empirically** (a real porter has
+had to do every one of these):
+
+- `strings` the binary / grep the install tree for hook event names, config
+  paths, and the instructions file it reads.
+- **Ask the running model to enumerate its own tool names** — e.g. "list the
+  exact machine names of every tool you can call." This is the authoritative way
+  to get tool names without inventing them (see Step 4).
+- Prove every assumption with a **unique-marker test**: inject a nonsense token
+  through the mechanism you think works, start a fresh session, and confirm the
+  token actually reached the model.
+
+**A fork does not inherit its parent's behavior.** A harness derived from another
+(e.g. a Gemini-derived CLI) may expose the parent's manifest fields and
+`@`-include syntax and *still not honor them the same way*. Verify with a marker;
+never assume the parent's recipe transfers.
+
+Then route to a shape:
+
+- Shell command at session start whose stdout is read → **Shape A**.
+- Plugin/extension module with lifecycle callbacks you run code in → **Shape B**.
+- Only ever an always-on instructions file, no hook and no code plugin →
+  **Shape C**.
+
+**Shapes compose — they are not mutually exclusive.** The *skill-discovery*
+mechanism and the *bootstrap* mechanism need not be the same shape — but **both
+must still ride the install mechanism** (rule 2). Decide the two questions
+separately: *where do skills get discovered?* and *how does the bootstrap reach
+the model every session?* A harness might install skills via a plugin yet need
+the bootstrap delivered another install-shipped way (an extension-declared
+context file, or — see below — by the harness surfacing the installed
+`using-superpowers` skill's own description at session start). If more than one
+install-mechanism surface injects automatically, prefer the most reliable. What
+you may **not** do is bridge a gap by editing the user's global config.
+
+### Shape A — Shell-hook
+
+The harness has a hook system that runs a shell command at session start and
+reads JSON from its stdout. The configured command runs `run-hook.cmd`, a
+polyglot wrapper that just locates bash and dispatches the named script; the
+script (`hooks/session-start`, or a harness-specific variant like
+`hooks/session-start-codex`) is what reads `using-superpowers/SKILL.md` and
+prints a JSON object whose **field name and nesting differ per harness**.
+
+- Reference: `hooks/session-start` (and `hooks/session-start-codex`),
+  `hooks/run-hook.cmd`, and the per-harness hook config `hooks/hooks.json`
+  (Claude Code), `hooks/hooks-codex.json` (Codex), `hooks/hooks-cursor.json`
+  (Cursor).
+- Manifests: `.codex-plugin/plugin.json`, `.cursor-plugin/plugin.json` point the
+  harness at `./skills/` and the right `hooks-*.json`. (Claude Code's
+  `.claude-plugin/plugin.json` sets neither field — it auto-discovers `skills/`
+  and `hooks/hooks.json` by convention.)
+
+> **A hook *system* is not a session-start *event*.** A harness can have a
+> `hooks.json` mechanism — and even contain the literal string `SessionStart` in
+> its binary — while having no hook event that fires at session start and can
+> inject context. (One real harness only exposed pre/post-tool and stop events;
+> the `SessionStart` strings were telemetry.) Confirm the *specific event* you
+> need exists and can write to the model's context before committing to Shape A.
+> If it can't, the bootstrap belongs in an instructions file (Shape C) instead.
+
+### Shape B — In-process plugin / extension
+
+The harness loads a JS/TS module that exposes lifecycle callbacks. You register
+the skills directory through the harness's API and inject the bootstrap by
+mutating the message array in code.
+
+- Reference: `.opencode/plugins/superpowers.js` (JavaScript) and
+  `.pi/extensions/superpowers.ts` (TypeScript). pi is the closest reference for
+  any harness that has **no native skill tool**.
+
+### Shape C — Instructions-file
+
+The harness has neither a shell hook nor a code plugin — its session-start
+surface is a context file that *your installed extension ships and the manifest
+declares* (e.g. Gemini's `contextFileName` → the extension's own `GEMINI.md`).
+You can't run code or mutate messages; the extension's context file points at the
+bootstrap. There is no injector to assemble a string or strip frontmatter — the
+harness loads the referenced content as-is. **This works only because the file is
+part of the installed extension** — never substitute "edit the user's global
+`GEMINI.md`/`AGENTS.md`" for shipping your own (rule 2).
+
+- Reference: `gemini-extension.json` (manifest, with `contextFileName`),
+  `GEMINI.md` (two `@`-includes — the bootstrap skill and the tool-mapping
+  reference), `skills/using-superpowers/references/gemini-tools.md`.
+- Note: `@`-include is a Gemini feature. If your harness loads an instructions
+  file but has no include syntax, you must inline the bootstrap content into the
+  file instead.
+- **Don't trust that an `@`-include is actually expanded — prove it.** A
+  Gemini-*derived* harness can accept `@./path` syntax yet treat it as a *hint
+  the model may choose to read* (it emits a file-read tool call) rather than a
+  guaranteed inline expansion. That's the difference between the bootstrap being
+  reliably present every session and the model maybe-reading it. Run a
+  unique-marker test: if the marker isn't in context *without* a tool call,
+  **inline the content** rather than `@`-include it.
+
+### Routing table
+
+| If the harness… | Use shape | Copy from |
+|---|---|---|
+| runs a shell command at session start and reads its stdout | A (shell-hook) | Codex (`hooks/session-start-codex` + `hooks/hooks-codex.json` + `.codex-plugin/`) |
+| is a JS/TS plugin host with session/message lifecycle callbacks | B (in-process) | OpenCode (`.opencode/`) — or pi (`.pi/`) if it has no native skill tool |
+| ships an extension-declared context file it always loads | C (instructions-file) | Gemini (`gemini-extension.json` + `GEMINI.md` + `references/gemini-tools.md`) |
+| has a plugin install command and a manifest `contextFileName` (or equivalent) the installer keeps | C via the plugin installer | Antigravity (`.antigravity-plugin/` — `agy plugin install` ships a generated context file; verify the installer preserves it — Part 6) |
+
+Most real harnesses fit one row cleanly; the last is the hybrid case (rule 2 still
+holds — the bootstrap rides the install mechanism, never a user-config edit).
+
+---
+
+## Part 5 — The porting procedure
+
+### Step 1 — Study the closest reference implementation
+
+Open the files named in Part 4 for your shape and read them end to end. The
+patterns below are summaries; the code is the spec.
+
+### Step 2 — Create the manifest / entry point
+
+Create whatever the harness uses to recognize the plugin. Match the existing
+ones in spirit:
+
+- **Shape A:** a `*-plugin/plugin.json` (see `.codex-plugin/plugin.json`) with
+  `name`, `version`, `description`, author/license/keywords, `"skills":
+  "./skills/"`, and `"hooks": "./hooks/hooks-<harness>.json"`. Plus the
+  `hooks-<harness>.json` itself, registering a session-start hook whose command
+  invokes `run-hook.cmd`.
+- **Shape B:** the module the harness loads (e.g. `.<harness>/plugins/*.js`) plus
+  whatever package metadata it needs to be discovered. The committed package
+  metadata is the **repo-root `package.json`**: `main` points at the OpenCode
+  plugin, the `pi` field (`pi.extensions`, `pi.skills`) plus the `pi-package`
+  keyword declare the pi extension. Per-harness local manifests and lockfiles are
+  kept out of git — `.opencode/.gitignore` excludes `node_modules`,
+  `package.json`, and lockfiles. Do the same for your harness's *local* install
+  artifacts so they don't pollute the repo — but never gitignore the repo-root
+  `package.json`, which is the tracked source of truth.
+  - **Build/dependency check.** Decide how the harness loads your module:
+    does it run the source directly (pi's `.ts` is referenced as-is from
+    `package.json`; OpenCode ships plain `.js`), or does it need a transpile/build
+    step? Superpowers is zero-runtime-dependency. pi's `import type
+    { ExtensionAPI }` works specifically because the harness runs the `.ts`
+    directly, supplies that type at load, and the repo never type-checks the file
+    in CI — the import isn't even declared as a dependency. If *your* harness
+    actually type-checks or bundles the plugin, that breaks: an undeclared type
+    import fails, and the PR rules only carve out *runtime* deps for new
+    harnesses, not dev/type packages. If you hit this, confirm the approach with
+    the maintainer rather than quietly adding a dependency. Keep any build output
+    out of git and document the command.
+- **Shape C (instructions-file):** a small manifest (see `gemini-extension.json`:
+  `name`, `description`, `version`, `contextFileName`) plus the context file
+  itself (`GEMINI.md` is just two `@`-includes: the bootstrap skill and the
+  tool-mapping reference). The Gemini manifest has no `skills` field — Gemini
+  auto-discovers the `skills/` directory bundled in the installed extension. If
+  your harness has a native skill tool but no manifest field to register the
+  directory, you must find its discovery convention (read its extension docs),
+  then verify empirically: after wiring, ask the model to list its available
+  skills — if the bundled skills don't appear, discovery isn't working yet.
+
+### Step 3 — Wire the bootstrap injection
+
+This is the heart of the port. The shared goal: at session start, get the
+`using-superpowers` skill content (wrapped in `<EXTREMELY_IMPORTANT>` tags) plus
+the harness's tool mapping in front of the model, with a note that the skill is
+already active so the model doesn't try to load it again. *How* you do that —
+and what you assemble vs. what the harness loads raw — depends entirely on your
+shape. Do **not** apply one shape's recipe to another.
+
+**Shape A — a script reads `SKILL.md` and prints the harness's JSON.** The
+dispatched script (`hooks/session-start`) `cat`s the whole `SKILL.md` (frontmatter
+included — that's fine; it's emitted verbatim), wraps it with the "You have
+superpowers… for all other skills use the Skill tool" preamble, escapes it, and
+prints the harness's JSON shape. The tool mapping for Shape A does **not** go
+inline here — it lives in `references/<harness>-tools.md` (Step 4). Get the JSON
+output shape exactly right. `hooks/session-start`
+detects the harness from environment variables and prints *one of three* shapes:
+
+- Cursor (`CURSOR_PLUGIN_ROOT` set): `{ "additional_context": "…" }`
+- Claude Code (`CLAUDE_PLUGIN_ROOT` set, `COPILOT_CLI` unset):
+  `{ "hookSpecificOutput": { "hookEventName": "SessionStart", "additionalContext": "…" } }`
+- Copilot CLI / SDK standard (else): `{ "additionalContext": "…" }`
+
+This is a trap. Emitting the wrong field, or an extra one, means the bootstrap
+either never injects or injects twice (Claude Code reads both
+`additional_context` and `hookSpecificOutput` without de-duplicating, so emitting
+both double-injects). Find the
+exact field, nesting, and event-matcher values your harness expects. Then
+decide: add a fourth branch to `hooks/session-start`, or — if the harness needs
+a different bootstrap message or env contract — add a dedicated
+`hooks/session-start-<harness>` script, the way Codex did. If you add a branch
+and your harness *also* sets an env var an earlier branch keys on (some harnesses
+set `CLAUDE_PLUGIN_ROOT` too), order your branch before the one that would
+otherwise shadow it. Match the harness's
+own event-matcher strings (Claude Code uses `startup|clear|compact`, Codex
+`startup|resume|clear`, Cursor `sessionStart`); wrong matchers mean the hook
+silently never fires.
+
+The **hook-config schema itself varies per harness** — don't assume the
+Claude/Codex shape is universal. Compare `hooks/hooks.json`,
+`hooks/hooks-codex.json`, and `hooks/hooks-cursor.json`: Cursor's uses
+`"version": 1`, a lowercase `sessionStart` key, a relative
+`./hooks/run-hook.cmd` command, and omits the `matcher`/`type`/`async` fields the
+others use. Match your `hooks-<harness>.json` to whichever existing file is
+closest, not to a single canonical template.
+
+The hook **command string references a harness-provided plugin-root variable**,
+and its name differs per harness: `hooks.json` uses `${CLAUDE_PLUGIN_ROOT}`,
+`hooks-codex.json` uses `${PLUGIN_ROOT}`, Cursor uses a relative path. Use
+whatever your harness exports. (The `session-start` script re-derives the root
+itself via `dirname`, so the script body doesn't depend on this — but the
+command in the manifest does.)
+
+**Discovering the harness's contract.** The three facts above — env var, JSON
+field/nesting, matcher strings — are the harness's contract, not Superpowers',
+so you have to source them. Read the harness's hook docs, or find out
+empirically: register a throwaway session-start hook that dumps its environment
+and emits a marker, then observe which env var identifies the harness and
+whether/how the harness ingests your stdout. Pin these down before writing the
+real branch.
+
+**Shape B — assemble the string in code, then inject as a user message.** Here
+you build the bootstrap yourself: read `SKILL.md`, strip its YAML frontmatter,
+and assemble `<EXTREMELY_IMPORTANT>` + a short preamble that the skill is already
+loaded and must not be re-invoked + the stripped body + the inline tool mapping +
+`</EXTREMELY_IMPORTANT>`. One subtlety the references disagree on: OpenCode's
+preamble says "do NOT use the skill tool…" (assumes a `skill` tool exists), while
+pi's just says "do not try to load using-superpowers again." If your harness has
+no skill tool, use pi's wording, not OpenCode's.
+
+Inject the result as a **user-role message, not a system message** — system
+messages bloat tokens when repeated every turn (#750) and multiple system
+messages break some models (#894). Three things you must replicate:
+
+- **Dedup guard.** The lifecycle callback can fire repeatedly (OpenCode's
+  transform runs on *every* agent step; pi's `context` fires per turn). Before
+  injecting, check whether a bootstrap marker is already present and skip if so.
+  (The references pick different markers — pi a custom string, OpenCode the
+  `EXTREMELY_IMPORTANT` tag; matching the tag is more robust since it needs no
+  harness-specific constant.) Cache the bootstrap content at module level so
+  you're not re-reading and re-parsing `SKILL.md` on every call (#1202).
+- **Compaction.** If the harness compacts/summarizes history, re-inject
+  afterward. pi sets an `injectBootstrap` flag on `session_start` and
+  `session_compact`, clears it on `agent_end`, and inserts the message *after*
+  any leading compaction-summary messages. OpenCode relies on its per-step
+  re-injection plus the dedup guard.
+- **Message-object shape is per-harness — discover yours, don't copy a literal.**
+  The two references use *incompatible* shapes: pi builds
+  `{ role, content: [{ type, text }], timestamp }`; OpenCode manipulates
+  `message.info.role` and `message.parts[]`. Find your harness's message shape
+  from its API; copying a reference's object literal verbatim will fail silently.
+
+**Shape C — point your extension's context file at the bootstrap; assemble
+nothing.** There is no injector, so you do *not* strip frontmatter or build a
+wrapped string. The context file your extension ships (declared by the manifest —
+*not* the user's own global file) pulls in two things: the `using-superpowers`
+skill and the harness's tool-mapping reference. `GEMINI.md`
+does this with two `@`-includes (`@./skills/using-superpowers/SKILL.md` and
+`@./skills/using-superpowers/references/<harness>-tools.md`); the harness loads
+them raw, frontmatter and all, and `SKILL.md` already carries its own
+`<EXTREMELY-IMPORTANT>` block internally. If your harness has no include syntax,
+inline the content into the instructions file instead. Gemini ships **no**
+"already loaded, don't re-invoke" preamble — for an `@`-include harness the
+content is the active instruction set, not a skill the model would re-load. If
+you find your harness does try to re-invoke, add that note as a literal line in
+the instructions file (you have no code to add it any other way).
+
+### Step 4 — Write the tool mapping
+
+Translate the action vocabulary into the harness's real tools. Cover every one
+of these actions (omit only what genuinely doesn't apply):
+
+- read a file
+- create / edit / delete a file (one `apply_patch`-style tool, or separate
+  write/edit?)
+- run a shell command
+- search file contents / find files by name (grep, glob)
+- fetch a URL / web search
+- **dispatch a subagent**, including how to pass the agent type — and any config
+  flag needed to enable it
+- **create / update todos** (treat older `TodoWrite` references as this action)
+- **invoke a skill** — see Step 5
+
+**Get the real tool names from the harness; never invent them.** If the docs
+don't list them, the authoritative source is the harness itself: in a live
+session, ask the model to "list the exact machine names of every tool you can
+call, one per line" and use what it reports.
+
+**How the harness finds the `skills/` directory is itself per-harness** — confirm
+it, don't assume. Possibilities: a manifest `skills` path field (Codex's
+`"skills": "./skills/"`); a *co-located* `skills/` the harness auto-scans (where a
+path field is **ignored** — one real harness only scanned a `skills/` sitting next
+to `plugin.json`); an API/registration call (OpenCode, pi); or you stage an
+install dir that pairs the manifest with a **symlink to the repo's `skills/`** and
+point the installer at the staging dir (verify the installer *dereferences* the
+symlink and copies the real files — confirm with `agy plugin validate`/`install`
+or the equivalent before relying on it). A `skills` path field is *not* portable.
+
+Where the mapping lives depends on shape:
+
+- **Shape A:** put it in `skills/using-superpowers/references/<harness>-tools.md`.
+  The agent reaches it from the bootstrap — `SKILL.md`'s "Platform Adaptation"
+  section links the per-harness references files. (Shape A harnesses have no
+  instructions file; the mapping is *not* inlined into the hook output.)
+- **Shape B:** the mapping is typically inlined into the bootstrap string you
+  inject (see the `toolMapping` constant in `superpowers.js`). pi keeps it in
+  *both* places — `piToolMapping()` inline **and** `references/pi-tools.md`. If
+  you maintain it in two places, update both, or the port is half-done.
+- **Shape C:** put it in `references/<harness>-tools.md` and pull it into the
+  always-loaded instructions file (e.g. `GEMINI.md` `@`-includes
+  `gemini-tools.md`).
+
+You may also add a one-line pointer to your harness in `SKILL.md`'s "Platform
+Adaptation" section so an agent reading the bootstrap knows where its mapping
+lives. This is the one edit to a `SKILL.md` a port may make — and only because
+that section is a pointer list, not behavior-shaping content. It does not violate
+the "don't edit skill bodies" rule (Part 1); do not touch anything else in any
+skill. (The list is a convenience pointer, not an exhaustive registry — not every
+harness is listed.)
+
+### Step 5 — Handle a harness with no native skill tool
+
+`using-superpowers/SKILL.md` tells the model to *never read skill files manually
+with file tools — always use your platform's skill-loading mechanism.* The point
+is "don't bypass the mechanism," not "never use file-read." What counts as "your
+platform's mechanism" depends on the harness — and for a harness with no skill
+tool, the documented mechanism *is* reading `SKILL.md`. So reading it there
+honors the rule rather than breaking it. Distinguish three cases:
+
+1. **Native `Skill`-style tool** (Claude Code, Copilot CLI, Gemini's
+   `activate_skill`): point the mapping at that tool.
+2. **Native skill *discovery* but no `Skill` tool** (pi, Antigravity): the harness
+   can find and list skills, but the model can't call a tool to load one. Get the
+   skills installed where the harness scans (pi registers via `resources_discover`
+   → `skillPaths`; OpenCode via its `config` hook; `agy plugin install` copies
+   them in), and tell the model to load a skill by **reading its `SKILL.md` with
+   the file-read tool when the skill applies** — the sanctioned mechanism here,
+   the way `references/pi-tools.md` states it.
+
+   **For the bootstrap itself, prefer a declared context file (Part 6).** If the
+   harness has a `contextFileName`-style manifest field — as Antigravity does —
+   ship a generated context file through the installer: it's guaranteed-loaded and
+   carries both the `using-superpowers` content and the tool mapping. That is the
+   strong, preferred path.
+
+   **Fallback — the surfaced skill index.** If there's no context-file field but
+   the harness surfaces each installed skill's name + description at session start,
+   you need *neither* a built index nor a runtime-list instruction — the harness
+   is the index, and `using-superpowers`'s own surfaced description can be what
+   triggers the model to load it. This is softer than a declared context file;
+   two things it does **not** give you, versus a context file / hook / in-process
+   injector — account for both:
+   - **It bootstraps *triggering*, not the *tool mapping*.** An injector prepends
+     `<harness>-tools.md` alongside `using-superpowers` every session. Here nothing
+     injects the mapping — the model only sees skill *descriptions* and must *read*
+     your `references/<harness>-tools.md` when it needs tool names. It works
+     because skills name actions (the model reads the mapping when it acts), but
+     it's softer than injection. Make sure the mapping is reachable from what the
+     model loads — e.g. linked from `SKILL.md`'s Platform Adaptation section and
+     installed alongside the skills — not just sitting in the repo.
+   - **There's no structural guarantee the trigger fires.** No `<EXTREMELY_IMPORTANT>`
+     wrapper, no dedup, no re-injection after compaction — firing depends on the
+     model choosing to act on a description it sees in the index. This is exactly
+     why the acceptance test is mandatory here: it is the *only* guarantee, so run
+     it on the model(s) your users will actually use, not just the strongest one.
+3. **No skill system at all:** there is nothing to register, and the *only*
+   mechanism is the model reading `SKILL.md` on demand. But the model can't read
+   what it can't find: `using-superpowers/SKILL.md` does **not** enumerate the
+   available skills, so on its own the model won't know which skills exist or
+   their triggers. You must supply a discovery path. Two options, and they differ
+   in durability: (a) generate a skill index (each `skills/*/SKILL.md`'s `name` +
+   `description` frontmatter) and place it *inside* the `<EXTREMELY_IMPORTANT>`
+   wrapper alongside the tool mapping (Shape B recipe above) so it's covered by
+   the dedup guard — but a build-time index goes stale as skills are added; or
+   (b) instruct the model to list `skills/*/SKILL.md` at runtime and read their
+   frontmatter to find a match — slower but never stale. Prefer (b) unless you
+   have a reason not to. Without either, a no-skill-system port loads the
+   bootstrap but silently never triggers any other skill.
+
+In cases 2 and 3, say plainly in your tool mapping that reading `SKILL.md` is the
+blessed path, so the model doesn't think it's violating the "never read skill
+files" rule. Don't go hunting for a `skillPaths`-style registration API in a
+harness that has no skill system — case 3 has none.
+
+### Step 6 — Add tests
+
+Match the existing per-harness test style:
+
+- **Shape A:** assert the hook's stdout has the exact JSON shape your harness
+  consumes, and that it contains the bootstrap. See `tests/hooks/test-session-start.sh`,
+  which validates each harness's output shape.
+- **Shape B:** a unit test that fakes the harness's plugin API and asserts the
+  lifecycle handlers register, the bootstrap injects once, the dedup guard
+  works, and (if relevant) compaction re-injection works. See
+  `tests/pi/test-pi-extension.mjs`. Add an isolated-install integration check in
+  the style of `tests/opencode/`.
+- If the bootstrap is cached, test that the cache behaves when the file is
+  missing (see the OpenCode caching tests).
+
+These automated tests cover the wiring; the live tmux run in Step 7 is what
+proves the integration actually triggers skills.
+
+### Step 7 — Install locally, then drive a live instance to verify
+
+You cannot confirm a port works by reading code. You have to run the harness with
+your in-progress port loaded and watch a real session — which is also how you
+produce the transcript the PR requires.
+
+**Install locally.** Point a *local* instance of the harness at your working
+tree, not a published build:
+
+- **Shape A / C:** install the plugin/extension from this repo's local path (or
+  symlink its directory into wherever the harness looks). Find the harness's
+  "install from a local directory / git checkout" path in its docs.
+- **Shape B:** register the local module — e.g. an `opencode.json` `plugin`
+  entry pointing at the local path, or pi resolving the `package.json` fields
+  from the repo.
+
+Reinstall after each change and restart the harness, since the bootstrap loads at
+startup.
+
+**Drive it with tmux.** Most harnesses are interactive REPLs/TUIs that can't be
+driven by piping stdin, so run the harness inside a detached tmux session and
+control it with `send-keys` / `capture-pane`. A harness may advertise a
+non-interactive "run one prompt" mode (e.g. `opencode run "..."`) — try it for the
+quick smoke check, but **don't depend on it**: these modes are frequently flaky,
+auth-gated, or trust-gated (one real harness's `--print` mode hung and timed out
+with no output every time). Be ready to do *everything*, including the smoke
+check, through tmux.
+
+**Clear the gates first, or tmux stalls silently.** Many harnesses block on
+first-run onboarding, a "do you trust this folder?" prompt, a sandbox mode, or a
+permission gate — and a detached tmux session will just sit there with no error
+while it waits. Before the run, pre-trust your scratch directory (in the harness's
+settings/config) or be prepared to answer those prompts via `send-keys`, and
+account for the harness's startup time in your first `sleep`.
+
+```bash
+# 1. Launch the harness detached, in a throwaway project dir
+mkdir -p /tmp/port-smoke
+tmux new-session -d -s port-test -c /tmp/port-smoke '<harness-launch-command>'
+
+# 2. Let it initialize — real TUIs take longer than you think (10s+ with a model
+#    handshake); tune this. THEN capture and clear any blocking modal before you
+#    type a prompt: first-run onboarding and "trust this folder?" are modal, so
+#    keystrokes sent during them select menu items instead of typing your prompt.
+sleep 12
+tmux capture-pane -t port-test -p          # onboarding / trust prompt? answer it via send-keys first
+# (e.g. tmux send-keys -t port-test Enter   # to accept a trust prompt — inspect before assuming)
+
+# 3. Smoke check: does the model know it has superpowers?
+#    Send the text and Enter as SEPARATE send-keys with a beat between them —
+#    sending them together races on some TUIs (Enter arrives before the text lands).
+tmux send-keys -t port-test 'What are your superpowers?'; sleep 0.4; tmux send-keys -t port-test Enter
+sleep 5
+tmux capture-pane -t port-test -p          # reply should show it knows its skills
+
+# 4. Acceptance test: exact prompt (note the escaped apostrophe), fresh session
+tmux send-keys -t port-test 'Let'\''s make a react todo list'; sleep 0.4; tmux send-keys -t port-test Enter
+# poll until the turn finishes — re-capture every few seconds, don't capture once
+sleep 8
+tmux capture-pane -t port-test -p          # PASS = brainstorming triggers BEFORE any code
+
+# 5. Save the transcript for the PR, then clean up
+tmux capture-pane -t port-test -p > /tmp/port-smoke/transcript.txt
+tmux kill-session -t port-test
+```
+
+tmux gotchas that bite here: wait after launch before the first capture; send the
+prompt text and `Enter` as *separate* `send-keys` calls with a short `sleep`
+between them (sending them together races on some TUIs), and `Enter` is a key name
+not `\n`; the agent's turn takes time, so **poll `capture-pane` in a loop** rather
+than capturing once; `capture-pane` shows only the visible pane, so for a long
+conversation use the harness's own transcript/log file as the record of truth;
+always `kill-session` when done.
+
+If the smoke check shows the model *doesn't* know it has superpowers, the
+bootstrap isn't loading — fix that before bothering with the acceptance test.
+
+---
+
+## Part 6 — Distribution and release
+
+A working integration in this repo isn't usable until a real user can install
+it. Distribution differs per harness ecosystem — find yours:
+
+| Channel | Example | What you do |
+|---|---|---|
+| Native plugin marketplace | Claude Code | Register in `.claude-plugin/marketplace.json`; users `/plugin install`. The external `superpowers-marketplace` repo is the source of truth users install from — see the release steps in `CLAUDE.md`. |
+| External marketplace fork, synced by script | Codex | `scripts/sync-to-codex-plugin.sh` rsyncs the tracked plugin files into a separate fork repo and opens a PR. Read its include/exclude list so you ship the right tree (it deliberately drops repo-internal dirs and other harnesses' dotdirs). |
+| Git-URL extension install | Gemini, OpenCode | Users install from a git URL (`gemini extensions install …`; an `opencode.json` `plugin` array entry). Document the exact command. |
+| Package-manifest fields | pi | Declared through fields in the repo-root `package.json`; users install via the harness's package command. |
+| Local installer (plugin install) | Antigravity (`agy`) | A small `install.sh` that runs the harness's own `agy plugin install` against a staging dir holding the manifest, the skills, and a generated `contextFileName` context file (the bootstrap). Everything arrives through the install mechanism — *not* by editing the user's config (see below). |
+
+Then:
+
+- **A plugin installer may silently strip *undeclared* files — so make the
+  bootstrap a file the installer *recognizes*, never a user-config edit.** A
+  `plugin install` typically copies only the components it knows about
+  (skills/agents/commands/mcp/hooks/context) and discards anything else, so a
+  context file the manifest doesn't declare just vanishes from the install. The
+  fix is **not** to give up and write into the user's config (**rule 2**) — it's
+  to declare the bootstrap as a recognized component. In escalation order:
+  - **Ship a context file the manifest declares.** If the harness has a
+    `contextFileName`-style field (an extension-declared file it loads every
+    session), that is the strongest clean bootstrap: declare it, and the installer
+    preserves it *and* the harness loads it. Generate it at install time from the
+    live `using-superpowers/SKILL.md` + the tool mapping (wrapped in
+    `<EXTREMELY_IMPORTANT>`) so the installed bootstrap never drifts. This is what
+    `.antigravity-plugin/install.sh` does — `agy plugin install` reports
+    `✔ context : ANTIGRAVITY.md`, and a clean session reads `using-superpowers`'s
+    SKILL.md, loads `brainstorming`, and enters the brainstorming flow before any
+    code. **Verify with a marker** that the installer keeps the file and the
+    harness loads it: one porter wrongly concluded it couldn't, because they
+    shipped the file *without* declaring `contextFileName` and it was stripped as
+    unrecognized.
+  - **Otherwise lean on the installed `using-superpowers` skill itself.** If the
+    harness surfaces each installed skill's name + description at session start,
+    the `using-superpowers` description ("Use when starting any conversation…")
+    can prompt the model to load it — installing the skill *is* the bootstrap.
+    Softer (no guaranteed wrapper; it carries triggering but not the tool mapping
+    — see Step 5), so prefer the declared context file when available.
+  - If neither works, the harness cannot be cleanly supported yet — **say so**
+    and raise it, rather than hand-editing the user's config.
+
+- **Write install docs.** A `docs/README.<harness>.md` and/or a
+  `.<harness>/INSTALL.md` (see `docs/README.opencode.md` and
+  `.opencode/INSTALL.md`), plus an install section in the top-level `README.md`.
+  The only supported install action is **running the harness's own install
+  command** (`agy plugin install`, `gemini extensions install`, `/plugin
+  install`, etc.). Hand-copying skill files and editing the user's global/personal
+  config are *both* off-limits (rule 2 / the PR rules). If the harness has no
+  install command at all — its only surface is a user-owned config file — then it
+  fails the "deliver via install mechanism" rule, and you should raise that rather
+  than ship an installer that edits the user's files.
+- **Register the version.** If your harness introduces a *new* versioned
+  manifest, add its path and version field to `.version-bump.json` so
+  `scripts/bump-version.sh` keeps it in lockstep (read that file to see what's
+  currently tracked). A new manifest that isn't registered there will ship a
+  stale version. If your harness instead rides an already-tracked file — pi
+  declares itself in the repo-root `package.json`, which is already listed —
+  there's nothing new to add.
+- **If no existing channel fits, you're standing up a new one.** None of the four
+  rows may match your harness. If it needs a Codex-style external fork sync,
+  `scripts/sync-to-codex-plugin.sh` is the template to clone (note its anchored
+  include/exclude list and its PR automation). And whenever you add a new
+  per-harness directory, add it to the *other* harnesses' sync excludes (e.g. the
+  EXCLUDES list in `sync-to-codex-plugin.sh`) so your dotdir doesn't leak into
+  their distributions.
+
+---
+
+## Part 7 — Cross-platform / Windows
+
+Only relevant to the shell-hook shape. `hooks/run-hook.cmd` is a polyglot: a
+single file that's valid as both a Windows batch script and a Unix shell script.
+On Windows, `cmd.exe` runs the batch portion, which locates `bash` (Git for
+Windows, then `bash` on PATH) and runs the named hook script; if no bash is
+found it exits cleanly so the harness still works, just without injection. On
+Unix, the leading `:` makes the batch block a no-op and the shell runs the
+script directly.
+
+Two rules this enforces, which you must respect:
+
+- **Hook scripts are extensionless** (`session-start`, not `session-start.sh`).
+  Claude Code's Windows handling prepends `bash` to any command containing
+  `.sh`, which would double-invoke. Name your hook script without an extension.
+- Don't write per-OS variants of the hook script. One extensionless bash script
+  plus the polyglot wrapper covers all three platforms.
+
+`hooks/run-hook.cmd` itself is the authoritative implementation — read it.
+(`docs/windows/polyglot-hooks.md` covers the background and rationale but
+describes an earlier per-script `.cmd`/`.sh` variant, so trust the code over that
+doc where they differ.)
+
+---
+
+## Part 8 — Submitting the PR
+
+- Target the **`dev`** branch. One harness per PR.
+- Fill in the PR template's **"New harness support"** section and paste the
+  complete acceptance-test transcript (the "Let's make a react todo list"
+  session showing `brainstorming` auto-triggering). A PR without this proof will
+  be closed.
+- Superpowers is a zero-dependency plugin. Don't add a third-party runtime
+  dependency. Adding a new harness is the one carve-out the contributor rules
+  allow, and even then keep it to what the integration strictly requires —
+  type-only imports that compile away are fine; runtime packages are not.
+- Don't touch skill bodies (Part 1). If you found yourself editing a `SKILL.md`
+  to make the port work, the fix belongs in your tool mapping instead.
+
+---
+
+## Appendix A — Reference integrations (current)
+
+Use this as the live index; when in doubt, read the files, not this table.
+
+| Harness | Entry point | Bootstrap mechanism | Tool mapping | Tests | Distribution |
+|---|---|---|---|---|---|
+| Claude Code | `.claude-plugin/plugin.json` + `hooks/hooks.json` | shell hook → `hooks/session-start` (`hookSpecificOutput.additionalContext`) | native `Skill` tool; `references/claude-code-tools.md` | `tests/hooks/` | marketplace |
+| Codex | `.codex-plugin/plugin.json` + `hooks/hooks-codex.json` | shell hook → `hooks/session-start-codex` | `references/codex-tools.md` | `tests/codex-plugin-sync/`, `tests/hooks/` | fork sync (`scripts/sync-to-codex-plugin.sh`) |
+| Cursor | `.cursor-plugin/plugin.json` + `hooks/hooks-cursor.json` | shell hook → `hooks/session-start` (`additional_context`) | `references/claude-code-tools.md` | `tests/hooks/` | hand-authored |
+| Copilot CLI | (shares Claude Code hook path; `COPILOT_CLI` env) | shell hook → `hooks/session-start` (`additionalContext`) | `references/copilot-tools.md` | `tests/hooks/` | — |
+| Gemini CLI | `gemini-extension.json` + `GEMINI.md` | instructions file `@`-includes bootstrap + mapping | `references/gemini-tools.md` | — | `gemini extensions install` |
+| OpenCode | `.opencode/plugins/superpowers.js` (declared via root `package.json` `main`) | in-process: `config` hook registers skills dir; `experimental.chat.messages.transform` injects user message | inline in `superpowers.js` | `tests/opencode/` | `opencode.json` plugin git URL |
+| pi | `.pi/extensions/superpowers.ts` | in-process: `resources_discover` registers skills; `context` event injects user message; lifecycle-flag + compaction-aware | `piToolMapping()` inline **and** `references/pi-tools.md` | `tests/pi/` | repo-root `package.json` fields |
+
+## Appendix B — Gotchas that have bitten porters
+
+- **Opt-in isn't a port.** If your human partner has to do anything per session
+  to get Superpowers, the acceptance test fails. Re-read Part 2.
+- **Wrong JSON field → silent failure or double injection.** Shape A only.
+  Confirm the exact field/nesting; Claude Code reads two fields without dedup.
+- **Hook-config schema varies per harness.** Shape A. Cursor's `hooks-cursor.json`
+  looks nothing like the Claude/Codex one (`version`, lowercase `sessionStart`,
+  relative command, no `matcher`/`type`/`async`). Match the closest existing file.
+- **Plugin-root env var differs per harness.** Shape A. The hook command uses
+  `${CLAUDE_PLUGIN_ROOT}` (Claude), `${PLUGIN_ROOT}` (Codex), or a relative path
+  (Cursor). Use what your harness exports; the script re-derives the root itself.
+- **System-message injection.** Shape B injects a *user* message on purpose
+  (#750, #894). Don't "fix" it to a system message.
+- **Per-step vs per-turn callbacks.** OpenCode fires every step (per-call dedup
+  guard); pi fires per turn (lifecycle flag + `agent_end` reset). Copying one
+  harness's dedup strategy onto the other's callback frequency breaks injection.
+- **Message-object shape is per-harness.** Shape B. pi and OpenCode use
+  incompatible shapes; discover yours, don't copy a reference's object literal.
+- **Hunting for a skill-registration API that doesn't exist.** A harness with no
+  skill system (not just no `Skill` tool) has nothing to register — the model
+  reads `SKILL.md` on demand. Don't assume a `skillPaths` equivalent exists.
+- **Mapping in two places.** For in-process plugins the mapping may live both
+  inline and in a `references/` file (pi). Update both.
+- **The "never read skill files" line.** It means "don't bypass your platform's
+  skill-loading mechanism," not "never use file-read." On a no-skill-tool harness
+  that mechanism *is* reading `SKILL.md` — say so explicitly in the mapping
+  (Part 5).
+- **`.sh` on Windows.** Keep hook scripts extensionless (Part 7).
+- **Unregistered version.** A new manifest not added to `.version-bump.json`
+  ships stale (Part 6).
+- **Editing skills to fit the harness.** Never. The fix goes in the tool mapping.

docs/superpowers/plans/2026-04-06-worktree-rototill.md

@@ -275,23 +275,16 @@ If no native tool is available, create a worktree manually using git.
 
 Follow this priority order:
 
-1. **Check existing directories:**
+1. **Check your instructions for a worktree directory preference.** If specified, use it without asking.
+
+2. **Check existing project-local directories:**
    ```bash
    ls -d .worktrees 2>/dev/null     # Preferred (hidden)
    ls -d worktrees 2>/dev/null      # Alternative
    ```
    If found, use that directory. If both exist, `.worktrees` wins.
 
-2. **Check for existing global directory:**
-   ```bash
-   project=$(basename "$(git rev-parse --show-toplevel)")
-   ls -d ~/.config/superpowers/worktrees/$project 2>/dev/null
-   ```
-   If found, use it (backward compatibility with legacy global path).
-
-3. **Check your instructions for a worktree directory preference.** If specified, use it without asking.
-
-4. **Default to `.worktrees/`.**
+3. **Default to `.worktrees/`.**
 
 #### Safety Verification (project-local directories only)
 
@@ -305,16 +298,11 @@ git check-ignore -q .worktrees 2>/dev/null || git check-ignore -q worktrees 2>/d
 
 **Why critical:** Prevents accidentally committing worktree contents to repository.
 
-Global directories (`~/.config/superpowers/worktrees/`) need no verification.
-
 #### Create the Worktree
 
 ```bash
-project=$(basename "$(git rev-parse --show-toplevel)")
-
 # Determine path based on chosen location
-# For project-local: path="$LOCATION/$BRANCH_NAME"
-# For global: path="~/.config/superpowers/worktrees/$project/$BRANCH_NAME"
+path="$LOCATION/$BRANCH_NAME"
 
 git worktree add "$path" -b "$BRANCH_NAME"
 cd "$path"
@@ -387,7 +375,6 @@ Ready to implement <feature-name>
 | `worktrees/` exists | Use it (verify ignored) |
 | Both exist | Use `.worktrees/` |
 | Neither exists | Check instruction file, then default `.worktrees/` |
-| Global path exists | Use it (backward compat) |
 | Directory not ignored | Add to .gitignore + commit |
 | Permission error on create | Sandbox fallback, work in place |
 | Tests fail during baseline | Report failures + ask |
@@ -464,7 +451,7 @@ git commit -m "feat: rewrite using-git-worktrees with detect-and-defer (PRI-974)
 Step 0: GIT_DIR != GIT_COMMON detection (skip if already isolated)
 Step 0 consent: opt-in prompt before creating worktree (#991)
 Step 1a: native tool preference (short, first, declarative)
-Step 1b: git worktree fallback with hooks symlink and legacy path compat
+Step 1b: git worktree fallback with project-local directory policy
 Submodule guard prevents false detection
 Platform-neutral instruction file references (#1049)"
 ```
@@ -663,7 +650,7 @@ WORKTREE_PATH=$(git rev-parse --show-toplevel)
 
 **If `GIT_DIR == GIT_COMMON`:** Normal repo, no worktree to clean up. Done.
 
-**If worktree path is under `.worktrees/` or `~/.config/superpowers/worktrees/`:** Superpowers created this worktree — we own cleanup.
+**If worktree path is under `.worktrees/` or `worktrees/`:** Superpowers created this worktree — we own cleanup.
 
 ```bash
 MAIN_ROOT=$(git -C "$(git rev-parse --git-common-dir)/.." rev-parse --show-toplevel)
@@ -707,7 +694,7 @@ git worktree prune  # Self-healing: clean up any stale registrations
 
 **Cleaning up harness-owned worktrees**
 - **Problem:** Removing a worktree the harness created causes phantom state
-- **Fix:** Only clean up worktrees under `.worktrees/` or `~/.config/superpowers/worktrees/`
+- **Fix:** Only clean up worktrees under `.worktrees/` or `worktrees/`
 
 **No confirmation for discard**
 - **Problem:** Accidentally delete work

docs/superpowers/plans/2026-05-07-pi-extension-and-evals.md

@@ -0,0 +1,143 @@
+# Pi Extension and Evals Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Add first-class Pi package support for Superpowers and add Pi as a Drill eval backend.
+
+**Architecture:** The Pi package is declared in the root `package.json` and loads existing `skills/` plus a small Pi extension. The extension injects the `using-superpowers` bootstrap into provider context as a user-role message on session startup and after compaction, with Pi-specific tool mapping. Drill gains a `pi` backend, Pi session-log normalization, and tests.
+
+**Tech Stack:** Pi TypeScript extension API, Node built-in test runner, Drill Python eval harness, pytest.
+
+---
+
+### Task 1: Pi package manifest and extension tests
+
+**Files:**
+- Modify: `package.json`
+- Create: `tests/pi/test-pi-extension.mjs`
+
+- [ ] **Step 1: Write failing package/extension tests**
+
+Create `tests/pi/test-pi-extension.mjs` with tests that import `extensions/superpowers.ts`, register fake Pi handlers, and assert:
+- root `package.json` has `keywords` containing `pi-package`
+- root `package.json` has `pi.skills: ["./skills"]`
+- root `package.json` has `pi.extensions: ["./extensions/superpowers.ts"]`
+- the extension registers `resources_discover`, `session_start`, `session_compact`, `context`, and `agent_end`
+- startup `context` injects exactly one user-role bootstrap message
+- `agent_end` clears startup injection
+- `session_compact` re-enables injection
+- the extension does not register `session_before_compact`
+
+- [ ] **Step 2: Run tests and verify RED**
+
+Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs`
+
+Expected: FAIL because `extensions/superpowers.ts` does not exist and `package.json` lacks the `pi` manifest.
+
+- [ ] **Step 3: Implement manifest fields**
+
+Update `package.json` with `description`, `keywords`, `pi.extensions`, and `pi.skills` while preserving existing `name`, `version`, `type`, and `main`.
+
+- [ ] **Step 4: Implement `extensions/superpowers.ts`**
+
+Create a zero-runtime-dependency extension that:
+- locates the package root from `import.meta.url`
+- reads `skills/using-superpowers/SKILL.md`
+- strips YAML frontmatter
+- appends Pi-specific tool mapping
+- exposes `resources_discover` with the skills path
+- marks bootstrap pending on `session_start` and `session_compact`
+- injects a user-role bootstrap message in `context`
+- inserts post-compact bootstrap after leading `compactionSummary` messages
+- clears pending bootstrap on `agent_end`
+
+- [ ] **Step 5: Run tests and verify GREEN**
+
+Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs`
+
+Expected: PASS.
+
+### Task 2: Pi tool mapping reference
+
+**Files:**
+- Create: `skills/using-superpowers/references/pi-tools.md`
+- Modify: `tests/pi/test-pi-extension.mjs`
+
+- [ ] **Step 1: Write failing test for Pi reference doc**
+
+Add assertions that `skills/using-superpowers/references/pi-tools.md` exists and documents mappings for `Skill`, `Task`, `TodoWrite`, and built-in tool names.
+
+- [ ] **Step 2: Run tests and verify RED**
+
+Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs`
+
+Expected: FAIL because `pi-tools.md` does not exist.
+
+- [ ] **Step 3: Add Pi reference doc**
+
+Create `skills/using-superpowers/references/pi-tools.md` explaining Pi-native skills, optional `pi-subagents`, no canonical todo/tasklist plugin, and built-in lowercase tools.
+
+- [ ] **Step 4: Run tests and verify GREEN**
+
+Run: `node --experimental-strip-types --test tests/pi/test-pi-extension.mjs`
+
+Expected: PASS.
+
+### Task 3: Drill Pi backend and session log normalization
+
+**Files:**
+- Create: `evals/backends/pi.yaml`
+- Modify: `evals/drill/backend.py`
+- Modify: `evals/drill/engine.py`
+- Modify: `evals/drill/normalizer.py`
+- Modify: `evals/tests/test_backend.py`
+- Modify: `evals/tests/test_normalizer.py`
+
+- [ ] **Step 1: Write failing backend/normalizer tests**
+
+Add pytest coverage for:
+- `load_backend("pi")` returns `family == "pi"`
+- Pi backend command starts with `pi` and includes `-e ${SUPERPOWERS_ROOT}`
+- `_resolve_log_dir()` for Pi points under `~/.pi/agent/sessions`
+- `filter_pi_logs_by_cwd()` keeps only session files whose header `cwd` matches the scenario workdir
+- `normalize_pi_logs()` extracts `toolCall` blocks from Pi assistant session entries and maps built-in lowercase tools to canonical names
+
+- [ ] **Step 2: Run tests and verify RED**
+
+Run: `uv run pytest evals/tests/test_backend.py evals/tests/test_normalizer.py -q`
+
+Expected: FAIL because the Pi backend and normalizer do not exist.
+
+- [ ] **Step 3: Add `evals/backends/pi.yaml`**
+
+Configure the backend to run `pi -e ${SUPERPOWERS_ROOT}`, use permissive TUI readiness, `/quit` shutdown, and Pi session log location.
+
+- [ ] **Step 4: Implement Pi family support**
+
+Update `Backend.family`, `Engine._resolve_log_dir`, `Engine._collect_tool_calls`, and `normalizer.py` with Pi log filtering and normalizing.
+
+- [ ] **Step 5: Run tests and verify GREEN**
+
+Run: `uv run pytest evals/tests/test_backend.py evals/tests/test_normalizer.py -q`
+
+Expected: PASS.
+
+### Task 4: Documentation and full verification
+
+**Files:**
+- Modify: `README.md`
+- Modify: `evals/README.md`
+
+- [ ] **Step 1: Document Pi install and eval backend**
+
+Add Pi to README quickstart/install list and add backend entry/usage to `evals/README.md`.
+
+- [ ] **Step 2: Run verification**
+
+Run:
+```bash
+node --experimental-strip-types --test tests/pi/test-pi-extension.mjs
+uv run pytest evals/tests/test_backend.py evals/tests/test_setup.py evals/tests/test_normalizer.py -q
+```
+
+Expected: all tests pass.

docs/superpowers/specs/2026-04-06-worktree-rototill-design.md

@@ -46,7 +46,7 @@ The skill describes the goal ("ensure work happens in an isolated workspace") an
 
 ### Provenance-based ownership
 
-Whoever creates the worktree owns its cleanup. If the harness created it, superpowers doesn't touch it. If superpowers created it (via git fallback), superpowers cleans it up. The heuristic: if the worktree lives under `.worktrees/` or `~/.config/superpowers/worktrees/`, superpowers owns it. Anything else (`.claude/worktrees/`, `~/.codex/worktrees/`, `.gemini/worktrees/`) belongs to the harness.
+Whoever creates the worktree owns its cleanup. If the harness created it, superpowers doesn't touch it. If superpowers created it (via git fallback), superpowers cleans it up. The heuristic: if the worktree lives under `.worktrees/` or `worktrees/`, superpowers owns it. Anything else (`.claude/worktrees/`, `~/.codex/worktrees/`, `.gemini/worktrees/`, or old user-global Superpowers paths) belongs to the harness or user and is left alone.
 
 ## Design
 
@@ -110,12 +110,11 @@ File splitting (Step 1b in a separate skill) was tested and proven unnecessary.
 When no native tool is available, create a worktree manually.
 
 **Directory selection** (priority order):
-1. Check for existing `.worktrees/` or `worktrees/` directory — if found, use it. If both exist, `.worktrees/` wins.
-2. Check for existing `~/.config/superpowers/worktrees/<project>/` directory — if found, use it (backward compatibility with legacy global path).
-3. Check the project's agent instruction file (CLAUDE.md, GEMINI.md, AGENTS.md, .cursorrules, or equivalent) for a worktree directory preference.
-4. Default to `.worktrees/`.
+1. Check the project's agent instruction file (CLAUDE.md, GEMINI.md, AGENTS.md, .cursorrules, or equivalent) for a worktree directory preference.
+2. Check for existing `.worktrees/` or `worktrees/` directory — if found, use it. If both exist, `.worktrees/` wins.
+3. Default to `.worktrees/`.
 
-No interactive directory selection prompt. The global path (`~/.config/superpowers/worktrees/`) is no longer offered as a choice to new users, but existing worktrees at that location are detected and used for backward compatibility.
+No interactive directory selection prompt. Old user-global Superpowers worktree paths are not detected or offered; new manual worktrees are project-local unless the user explicitly specifies another location.
 
 **Safety verification** (project-local directories only):
 
@@ -232,7 +231,7 @@ if GIT_DIR == GIT_COMMON:
     # Normal repo, no worktree to clean up
     done
 
-if worktree path is under .worktrees/ or ~/.config/superpowers/worktrees/:
+if worktree path is under .worktrees/ or worktrees/:
     # Superpowers created it — we own cleanup
     cd to main repo root       # Bug #238 fix
     git worktree remove <path>
@@ -318,7 +317,7 @@ As of 2026-04-06, Claude Code is the only harness with an agent-callable mid-ses
 
 ### Provenance heuristic
 
-The `.worktrees/` or `~/.config/superpowers/worktrees/` = ours, anything else = hands off` heuristic works for every current harness. If a future harness adopts `.worktrees/` as its convention, we'd have a false positive (superpowers tries to clean up a harness-owned worktree). Similarly, if a user manually runs `git worktree add .worktrees/experiment` without superpowers, we'd incorrectly claim ownership. Both are low risk — every harness uses branded paths, and manual `.worktrees/` creation is unlikely — but worth noting.
+The `.worktrees/` or `worktrees/` = ours, anything else = hands off` heuristic works for every current harness. If a future harness adopts one of those project-local directories as its convention, we'd have a false positive (superpowers tries to clean up a harness-owned worktree). Similarly, if a user manually runs `git worktree add .worktrees/experiment` without superpowers, we'd incorrectly claim ownership. Both are low risk — every harness uses branded paths, and manual `.worktrees/` creation is unlikely — but worth noting.
 
 ### Detached HEAD finishing
 

docs/superpowers/specs/2026-05-05-platform-neutral-config-refs-design.md

@@ -0,0 +1,77 @@
+# Platform-neutral config-file references — Phase B design
+
+## Background
+
+Phase A (see `2026-05-05-platform-neutral-prose-design.md`) replaced generic third-person "Claude" prose with agent-neutral forms. This phase tackles the next category: references to the per-platform instruction file (CLAUDE.md, AGENTS.md, GEMINI.md) inside skills.
+
+The plugin runs on multiple harnesses, and each one reads its own instruction file. Where a skill names CLAUDE.md as if it were the only file, that's a Claude-Code-centric assumption that doesn't hold on Codex / Gemini CLI / OpenCode.
+
+## In scope
+
+Two specific lines in active skills:
+
+1. **`skills/writing-skills/SKILL.md:58`** — `Project-specific conventions (put in CLAUDE.md)`
+2. **`skills/receiving-code-review/SKILL.md:30`** — `"You're absolutely right!" (explicit CLAUDE.md violation)`
+
+## Out of scope
+
+- **`skills/using-superpowers/SKILL.md:22, 26`** — instruction-priority list. The list already names all three (CLAUDE.md, GEMINI.md, AGENTS.md) inclusively, which is correct: the section is making a real claim about *what counts as user instruction* on a multi-platform plugin. No change needed.
+- **Historical / example artifacts**:
+  - `skills/systematic-debugging/CREATION-LOG.md` — attribution path (`~/.claude/CLAUDE.md`) is a historical fact.
+  - `skills/writing-skills/examples/CLAUDE_MD_TESTING.md` — the entire file is a worked example testing CLAUDE.md content variants. The filename, body, and the reference from `testing-skills-with-subagents.md` all stay; normalizing them defeats the example.
+- **Platform-tooling references** — Phase D candidates:
+  - `skills/using-superpowers/SKILL.md:40` (Gemini CLI tool mapping note about GEMINI.md)
+  - `skills/using-superpowers/references/gemini-tools.md` (`save_memory` persists to GEMINI.md)
+
+## Substitution rules
+
+Two distinct calls, one per in-scope line.
+
+### Rule 1: "where to put project-specific conventions"
+
+`writing-skills/SKILL.md:58`:
+
+- **Before:** `Project-specific conventions (put in CLAUDE.md)`
+- **After:** `Project-specific conventions (put in your instructions file)`
+
+Use a generic phrase rather than picking one filename. Different harnesses read different files (CLAUDE.md, AGENTS.md, GEMINI.md, etc.) and the skill should not assume one. The platform-tools reference docs (`references/{codex,copilot,gemini}-tools.md`) are the right place to name each platform's preferred file.
+
+### Rule 2: the "(explicit CLAUDE.md violation)" parenthetical
+
+`receiving-code-review/SKILL.md:30`:
+
+- **Before:** `"You're absolutely right!" (explicit CLAUDE.md violation)`
+- **After:** `"You're absolutely right!" (explicit instruction-file violation)`
+
+The parenthetical is doing real work — it signals this phrase isn't just stylistically bad, it actively violates rules many users put in their instruction files. "Instruction file" is the natural cross-platform term covering AGENTS.md / CLAUDE.md / GEMINI.md collectively, and keeps the original signal without picking one filename or softening to "common".
+
+## Commit plan
+
+Atomic commits, in order:
+
+1. **`writing-skills/SKILL.md`** — CLAUDE.md → "your instructions file" in the "where to put project conventions" line
+2. **`receiving-code-review/SKILL.md`** — CLAUDE.md → instruction-file in the violation parenthetical
+3. **Platform-tools reference docs** — add the preferred per-platform instructions filename (CLAUDE.md, AGENTS.md, GEMINI.md, etc.) to each `references/{codex,copilot,gemini}-tools.md` so readers can resolve "your instructions file" to a real filename.
+
+Each commit message names "Phase B" and the slice.
+
+## Verification
+
+After each commit:
+
+- Read the surrounding paragraph to confirm grammar and meaning still parse.
+- `grep -n "CLAUDE\.md" <touched-file>` — no remaining hits in active prose (carve-outs already documented).
+
+After both commits:
+
+- `grep -rn "CLAUDE\.md" skills/` should return only the documented carve-outs (CREATION-LOG, CLAUDE_MD_TESTING and its inbound reference, the priority list in using-superpowers).
+
+## Non-goals
+
+- Do not touch the priority list ordering in `using-superpowers/SKILL.md`. Reordering CLAUDE.md / GEMINI.md / AGENTS.md is an aesthetic change, not a substitution, and out of scope here.
+- Do not rename `examples/CLAUDE_MD_TESTING.md` or change its content.
+- Do not modify Gemini-CLI-specific tooling references (Phase D candidates).
+
+## Implementation note
+
+Phase B as written here covered three commits and the three non-Claude-Code platform-tools refs. Implementation went one step further: a fourth ref, `references/claude-code-tools.md`, was added in commit `8505703` for symmetry, so Claude Code's instructions-file conventions and tool-name list live alongside the others rather than implicitly in the surrounding skill prose. That addition wasn't anticipated in this spec but is consistent with its intent.

docs/superpowers/specs/2026-05-05-platform-neutral-prose-design.md

@@ -0,0 +1,94 @@
+# Platform-neutral prose — Phase A design
+
+## Background
+
+Superpowers ships to multiple agent runtimes (Claude Code, Codex, Cursor, OpenCode, Copilot CLI, Gemini CLI). Skill content and supporting docs were written first for Claude Code and use "Claude" in places where any runtime's agent applies. OpenAI's vendored fork (openai/plugins#217) attempted a wholesale rewrite that was actively wrong in places — rewriting historical attribution paths, model names, and platform-specific install instructions — and we want to avoid that mistake while still removing platform-centric prose where it is genuinely incidental.
+
+The full effort is broken into phases by reference category. **This spec covers Phase A only:** generic third-person prose mentioning "Claude" in non-platform-specific contexts. Later phases (config-file references, marketing copy, tool-name references) are out of scope here and will get their own specs.
+
+## In scope
+
+Generic prose mentions of "Claude" in:
+
+- `skills/*/SKILL.md` and supporting `.md` files in active skill directories
+- `skills/writing-skills/anthropic-best-practices.md`
+- `README.md` (only where the mention is generic prose, not platform marketing)
+
+Plus one coined-term rename: **Claude Search Optimization (CSO) → Skill Discovery Optimization (SDO)** in `skills/writing-skills/SKILL.md`.
+
+## Out of scope
+
+- **Platform/runtime statements** — "In Claude Code:", install instructions, tool-mapping references. (Phase D candidate.)
+- **Config-file references** — CLAUDE.md, AGENTS.md, GEMINI.md priority lists and "where to put project conventions" callouts. (Phase B.)
+- **Tool-name references** — `Skill`, `Bash`, `Read`, `Task`, `TodoWrite`. Skills are written in Claude Code's tool vocabulary; the existing `references/{codex,copilot,gemini}-tools.md` files map them. (At the time this spec was written, the plan was to defer or skip these. Phase E ended up doing them — replacing tool names with action language across active skills and unifying the platform-tools refs around the same vocabulary.)
+- **Marketing copy** in README — "Superpowers for Claude Code", platform-named install sections. (Phase C.)
+- **Historical artifacts** — `docs/plans/*.md`, `docs/superpowers/specs/*.md`, `CREATION-LOG.md`. These are dated, point-in-time documents; rewriting them rewrites history.
+- **Model identifiers** — Claude Haiku / Sonnet / Opus. These are real product names.
+- **Filename / URL references** — `CLAUDE.md`, `claude.com`, `claude-plugin/`, paths under `~/.claude/`.
+- **`anthropic-best-practices.md` filename** — the file remains named after its source even though we rewrite the prose inside it.
+
+## Replacement style
+
+Use a mix that reads naturally in English:
+
+- **Second person — "your agent"** when addressing the skill author about *their* runtime
+  - "your agent reads the description"
+- **Third person — "the agent" / "agents" / "an agent"** when describing system behavior generically
+  - "Future agents find your skills"
+  - "Use words an agent would search for"
+  - "Agents read SKILL.md only when the skill becomes relevant"
+
+Pick whichever fits the surrounding sentence; do not force consistency at the cost of awkward phrasing. Pluralize when natural ("future agents", "agents read") rather than always saying "the agent".
+
+### Carve-outs that stay as "Claude"
+
+- Model names: Claude Haiku, Claude Sonnet, Claude Opus
+- Filenames and URLs: `CLAUDE.md`, `claude.com`, `~/.claude/`
+- Branded platform name "Claude Code" wherever it refers to the runtime as such (handled in later phases)
+
+### Coined-term rename
+
+- **Claude Search Optimization (CSO) → Skill Discovery Optimization (SDO)**
+  - Appears in `skills/writing-skills/SKILL.md` as a section heading and in nearby prose. Rename the heading, the acronym, and any in-file cross-references.
+
+## Files affected
+
+Approximate counts based on a `grep` filtered to exclude carve-outs:
+
+| File | Generic-prose mentions |
+|------|------------------------|
+| `skills/writing-skills/SKILL.md` | ~12 (includes CSO heading + body) |
+| `skills/writing-skills/anthropic-best-practices.md` | ~30 |
+| `skills/writing-skills/examples/CLAUDE_MD_TESTING.md` | ~1 — filename stays (it's a CLAUDE.md test artifact); the "Variant C: Claude.AI Emphatic Style" heading also stays (it's a label naming a specific style) |
+| `README.md` | ~1 |
+
+Final list confirmed during implementation by re-running the filtered grep.
+
+## Commit plan
+
+Four atomic commits, in order:
+
+1. **Rename CSO → SDO** in `skills/writing-skills/SKILL.md`. Mechanical, isolated, easy to revert if we change our minds about the term.
+2. **Active skills prose** — generic "Claude" → "agent" forms across `skills/*/SKILL.md` and supporting `.md`, excluding `anthropic-best-practices.md`.
+3. **`anthropic-best-practices.md` prose** — same substitution rules. Separate commit because this file is a vendored adaptation of an external doc; isolating the change makes future reconciliation with upstream easier to read.
+4. **README.md prose** *(only if any generic-prose mentions remain after filtering)*. Skipped if empty.
+
+Each commit message names the phase ("Phase A") and the slice ("rename CSO to SDO", "agent prose in active skills", etc.) so the series is self-documenting.
+
+## Verification
+
+After each commit:
+
+- `grep -rn "Claude" <touched-paths>` — every remaining hit must fall into a documented carve-out (model name, filename, URL, "Claude Code" platform name, historical artifact).
+- Read the touched file end-to-end — substitutions should not have broken sentence flow, pronoun agreement, or list parallelism.
+- No tests to run; this is prose-only.
+
+After the final commit:
+
+- Skim each modified skill in a live session to confirm nothing reads awkwardly.
+
+## Non-goals
+
+- Do not change behavior, structure, headings (other than CSO→SDO), examples, code blocks, or YAML frontmatter.
+- Do not introduce new sections, callouts, or compatibility notes.
+- Do not "improve" prose beyond the substitution while editing.

docs/superpowers/specs/2026-05-05-platform-neutral-readme-design.md

@@ -0,0 +1,47 @@
+# Platform-neutral README ordering — Phase C design
+
+## Background
+
+Phases A and B (see `2026-05-05-platform-neutral-prose-design.md` and `2026-05-05-platform-neutral-config-refs-design.md`) already neutralized generic Claude prose and config-file references in the README. The remaining platform-leaning signal is layout: the README's two platform listings put Claude Code first and aren't strictly alphabetical elsewhere.
+
+This phase fixes the ordering. No prose changes.
+
+## In scope
+
+1. **Quickstart platform list** (`README.md:7`) — the inline link list of supported harnesses
+2. **Installation section ordering** (`README.md:35–152`) — the per-harness install sub-sections
+
+## Out of scope
+
+- Prose, marketplace names, plugin IDs, URLs — all factually correct as-is.
+- Visual weight of the Claude Code section (which has two sub-sections — official Anthropic marketplace and Superpowers marketplace). Both are real install paths; collapsing them would hide accurate info.
+- Section headings and content within each install block — only the ordering of the blocks changes.
+
+## Substitution
+
+Both listings reorder to strict alphabetical:
+
+| Old order | New order |
+|-----------|-----------|
+| Claude Code | Claude Code |
+| Codex CLI | Codex App |
+| Codex App | Codex CLI |
+| Factory Droid | Cursor |
+| Gemini CLI | Factory Droid |
+| OpenCode | Gemini CLI |
+| Cursor | GitHub Copilot CLI |
+| GitHub Copilot CLI | OpenCode |
+
+Three moves: Codex App swaps with Codex CLI; Cursor moves up two slots; GitHub Copilot CLI moves up one.
+
+Claude Code remains first by alphabetical chance (`Cl…` precedes `Co…`).
+
+## Commit plan
+
+One atomic commit covering both listings, since changing one without the other would create inconsistency between the quickstart and the installation section.
+
+## Verification
+
+- Quickstart anchors (`#claude-code`, `#codex-app`, etc.) still resolve to existing `### …` headings — no headings renamed.
+- Each install sub-section's body is byte-identical pre/post; only positions changed.
+- `git diff README.md` shows section moves only, no content edits.

docs/testing.md

@@ -14,7 +14,7 @@ Live in `tests/`. Currently:
 - `tests/codex-plugin-sync/` — bash sync verification.
 - `tests/claude-code/test-helpers.sh`, `analyze-token-usage.py` — utilities used by remaining bash tests.
 - `tests/claude-code/test-subagent-driven-development.sh` — agent-can-describe-SDD test (no drill counterpart; tests description-recall, not behavior).
-- `tests/claude-code/test-subagent-driven-development-integration.sh` — extended SDD integration with token analysis (drill covers the YAGNI subset; bash adds commit-count, TodoWrite, and token telemetry assertions).
+- `tests/claude-code/test-subagent-driven-development-integration.sh` — extended SDD integration with token analysis (drill covers the YAGNI subset; bash adds commit-count, Claude Code task-tracking, and token telemetry assertions).
 - `tests/claude-code/test-worktree-native-preference.sh` — RED-GREEN-REFACTOR validation for worktree skill (drill covers the PRESSURE phase; bash also covers RED/GREEN baselines).
 - `tests/explicit-skill-requests/` — Haiku-specific, multi-turn, and skill-name-prompted tests not covered by drill.
 

evals/.gitignore

@@ -1,9 +0,0 @@
-results/
-__pycache__/
-*.pyc
-*.egg-info/
-dist/
-build/
-.venv/
-.env
-.claude/

evals/CLAUDE.md

@@ -1,46 +0,0 @@
-# Drill
-
-Superpowers skill compliance benchmark. Python 3.11+, managed with uv.
-
-## Commands
-
-- **install**: `uv sync --extra dev`
-- **test**: `uv run pytest`
-- **test single**: `uv run pytest tests/test_engine.py -x -q`
-- **lint**: `uv run ruff check`
-- **format**: `uv run ruff format`
-- **typecheck**: `uv run ty check`
-- **run scenario**: `uv run drill run <scenario> -b <backend>`
-- **sweep**: `uv run drill run <scenario> --models claude-opus-4-6,claude-opus-4-7 --n 10`
-- **compare**: `uv run drill compare <scenario>`
-- **list**: `uv run drill list`
-
-## Architecture
-
-- `drill/engine.py` — Tmux session orchestration. Creates workdir, runs setup helpers, drives actor/agent turns, collects results.
-- `drill/actor.py` — Sonnet 4.6 LLM simulating a user. Reads turn intents from scenario YAML and generates realistic prompts.
-- `drill/verifier.py` — Sonnet 4.6 LLM evaluating session transcript + filesystem against semantic criteria.
-- `drill/assertions.py` — Deterministic post-session checks. Runs shell commands from `verify.assertions` in the results dir.
-- `drill/sweep.py` — Multi-backend, N-repetition orchestrator. Wraps Engine with try/except per run, writes run-group.json manifest.
-- `drill/compare.py` — Loads results, computes pass rates and Wilson CIs, formats comparison tables.
-- `drill/stats.py` — Wilson score confidence interval for pass rate estimation at small N.
-- `scenarios/*.yaml` — Scenario definitions (setup, turns, limits, verify).
-- `setup_helpers/*.py` — Repo fixture creators. Each creates a git repo with specific conditions.
-- `backends/*.yaml` — Per-backend CLI config (args, env, idle patterns, shutdown commands).
-- `bin/` — Assertion helper scripts: `tool-called`, `tool-not-called`, `tool-count`, `tool-before`, `tool-arg-match`. Run against `tool_calls.jsonl` in results dir.
-
-## Conventions
-
-- Setup helpers take `workdir: Path` and mutate the filesystem. Register in `setup_helpers/__init__.py`.
-- Scenarios use `user_posture: naive` (no skill names) or `spec-aware` (can name skills).
-- Verify criteria are semantic (LLM-evaluated). Verify assertions are deterministic (exit code 0 = pass).
-- Assertions run in the results dir with `$DRILL_WORKDIR` pointing to the scenario workdir and `bin/` on PATH.
-- Backend YAMLs are fully self-contained — no override/alias system.
-
-## Required env
-
-```
-ANTHROPIC_API_KEY=sk-...
-```
-
-`SUPERPOWERS_ROOT` defaults to the parent of `evals/` (the superpowers repo root). Override only if running drill against a different superpowers checkout.

evals/README.md

@@ -1,113 +0,0 @@
-# Drill
-
-Superpowers skill compliance benchmark. Drives AI coding agents through
-tmux sessions and evaluates whether they follow superpowers workflows
-correctly.
-
-## How it works
-
-1. **Setup** — a helper creates a git repo with specific conditions (worktree state, plan files, code fixtures)
-2. **Actor** — a Sonnet 4.6 LLM plays the user, following turn intents from the scenario YAML
-3. **Agent** — the backend under test (Claude Code, Codex, Gemini CLI) runs in a real tmux session
-4. **Verifier** — a Sonnet 4.6 LLM evaluates the session transcript + filesystem against criteria
-5. **Assertions** — deterministic checks (tool-called, tool-count, shell commands) run post-session
-
-## Setup
-
-```bash
-uv sync --extra dev
-```
-
-Optional git hooks:
-```bash
-uv --project evals run pre-commit install
-uv --project evals run pre-commit run --all-files
-```
-
-Required environment:
-```bash
-export ANTHROPIC_API_KEY=sk-...
-```
-
-`SUPERPOWERS_ROOT` defaults to the parent of `evals/` (the superpowers repo root) and only needs to be set if you're running drill against a different superpowers checkout.
-
-## Usage
-
-```bash
-# Run a single scenario on a single backend
-uv run drill run worktree-creation-from-main -b claude
-
-# Run with N repetitions
-uv run drill run spec-writing-blind-spot -b claude-opus-4-6 --n 5
-
-# Sweep across multiple backends
-uv run drill run spec-writing-blind-spot --models claude-opus-4-6,claude-opus-4-7 --n 10
-
-# Compare results
-uv run drill compare spec-writing-blind-spot
-
-# List available scenarios
-uv run drill list
-```
-
-## Scenarios
-
-| Category | Scenarios | Tests |
-|----------|-----------|-------|
-| Worktree | 11 scenarios | Worktree creation, detection, consent, detached HEAD, and native-tool pressure |
-| Skill triggering | 6 scenarios | Auto-invocation for core Superpowers skills |
-| SDD workflow | 5 scenarios | Explicit invocation, mid-conversation invocation, real-project execution, and YAGNI enforcement |
-| Review/spec/verification | 6 scenarios | Code review, spec review, architectural targeting, design blind spots, and verification reflexes |
-| Tool mapping | 3 scenarios | Codex and Gemini subagent tool-name mapping |
-
-## Backends
-
-| Backend | CLI | Model |
-|---------|-----|-------|
-| `claude` | Claude Code | opus-4-7 (default) |
-| `claude-opus-4-6` | Claude Code | opus-4-6 |
-| `claude-opus-4-7` | Claude Code | opus-4-7 |
-| `claude-opus-4-6-1m` | Claude Code | opus-4-6 (1M context) |
-| `claude-opus-4-7-1m` | Claude Code | opus-4-7 (1M context) |
-| `codex` | Codex CLI | — |
-| `gemini` | Gemini CLI | auto-gemini-3 |
-| `gemini-2-5-flash` | Gemini CLI | gemini-2.5-flash |
-
-## Project structure
-
-```
-drill/              # Core engine
-  cli.py            # Click CLI (run, compare, list)
-  engine.py         # Tmux session orchestration
-  actor.py          # User-simulator LLM
-  verifier.py       # Criteria evaluator LLM
-  assertions.py     # Deterministic post-session assertions
-  compare.py        # Result loading and cross-backend comparison
-  sweep.py          # Multi-backend N-rep orchestrator
-  stats.py          # Wilson score confidence intervals
-scenarios/          # YAML scenario definitions
-setup_helpers/      # Repo fixture creators
-backends/           # Per-backend YAML configs
-bin/                # Assertion helper scripts (tool-called, tool-count, etc.)
-prompts/            # Actor and verifier system prompts
-fixtures/           # Static template repos
-tests/              # pytest suite (122 tests)
-docs/               # Design spec and manual testing guide
-```
-
-## Tests
-
-```bash
-uv run pytest
-uv run ruff check
-uv run ty check
-```
-
-## Writing a new scenario
-
-1. Create a setup helper in `setup_helpers/` if you need a custom fixture
-2. Register it in `setup_helpers/__init__.py`
-3. Create `scenarios/your-scenario.yaml` with setup, turns, limits, and verify sections
-4. Run it: `uv run drill run your-scenario -b claude`
-
-See [docs/design.md](docs/design.md) for the full design spec.

evals/backends/claude-haiku.yaml

@@ -1,26 +0,0 @@
-name: claude-haiku
-cli: claude
-args:
-  - "--dangerously-skip-permissions"
-  - "--plugin-dir"
-  - "${SUPERPOWERS_ROOT}"
-  - "--model"
-  - "haiku"
-required_env:
-  - ANTHROPIC_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run: []
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 3
-  ready_pattern: "^❯|^\\$|Human:|Enter to confirm"
-busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]"
-max_busy_seconds: 1800
-startup_timeout: 60
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.claude/projects/**/session-*.jsonl"

evals/backends/claude-opus-4-6-1m.yaml

@@ -1,26 +0,0 @@
-name: claude-opus-4-6-1m
-cli: claude
-args:
-  - "--dangerously-skip-permissions"
-  - "--plugin-dir"
-  - "${SUPERPOWERS_ROOT}"
-  - "--model"
-  - "claude-opus-4-6[1m]"
-required_env:
-  - ANTHROPIC_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run: []
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 3
-  ready_pattern: "^❯|^\\$|Human:|Enter to confirm"
-busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]"
-max_busy_seconds: 1800
-startup_timeout: 60
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.claude/projects/**/session-*.jsonl"

evals/backends/claude-opus-4-6.yaml

@@ -1,26 +0,0 @@
-name: claude-opus-4-6
-cli: claude
-args:
-  - "--dangerously-skip-permissions"
-  - "--plugin-dir"
-  - "${SUPERPOWERS_ROOT}"
-  - "--model"
-  - "claude-opus-4-6"
-required_env:
-  - ANTHROPIC_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run: []
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 3
-  ready_pattern: "^❯|^\\$|Human:|Enter to confirm"
-busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]"
-max_busy_seconds: 1800
-startup_timeout: 60
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.claude/projects/**/session-*.jsonl"

evals/backends/claude-opus-4-7-1m.yaml

@@ -1,26 +0,0 @@
-name: claude-opus-4-7-1m
-cli: claude
-args:
-  - "--dangerously-skip-permissions"
-  - "--plugin-dir"
-  - "${SUPERPOWERS_ROOT}"
-  - "--model"
-  - "claude-opus-4-7[1m]"
-required_env:
-  - ANTHROPIC_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run: []
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 3
-  ready_pattern: "^❯|^\\$|Human:|Enter to confirm"
-busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]"
-max_busy_seconds: 1800
-startup_timeout: 60
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.claude/projects/**/session-*.jsonl"

evals/backends/claude-opus-4-7.yaml

@@ -1,26 +0,0 @@
-name: claude-opus-4-7
-cli: claude
-args:
-  - "--dangerously-skip-permissions"
-  - "--plugin-dir"
-  - "${SUPERPOWERS_ROOT}"
-  - "--model"
-  - "claude-opus-4-7"
-required_env:
-  - ANTHROPIC_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run: []
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 3
-  ready_pattern: "^❯|^\\$|Human:|Enter to confirm"
-busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]"
-max_busy_seconds: 1800
-startup_timeout: 60
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.claude/projects/**/session-*.jsonl"

evals/backends/claude.yaml

@@ -1,32 +0,0 @@
-name: claude
-cli: claude
-args:
-  - "--dangerously-skip-permissions"
-  - "--plugin-dir"
-  - "${SUPERPOWERS_ROOT}"
-  - "--model"
-  - "opus"
-required_env:
-  - ANTHROPIC_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run: []
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 3
-  ready_pattern: "^❯|^\\$|Human:|Enter to confirm"
-# Matches when Claude is actively working — spinners, "Thinking", time counter,
-# or "esc to cancel". Engine extends its wait deadline when any of these match
-# so the Actor doesn't interrupt long-running subagent work.
-busy_pattern: "esc to cancel|Thinking\\.\\.\\.|\\(esc to cancel[^)]*\\)|[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]"
-# Maximum total seconds the engine will extend the deadline across all busy
-# detections during a single _wait_for_ready call. Long-running subagent work
-# can take a while, so 30 minutes gives plenty of headroom.
-max_busy_seconds: 1800
-startup_timeout: 60
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.claude/projects/**/session-*.jsonl"

evals/backends/codex.yaml

@@ -1,20 +0,0 @@
-name: codex
-cli: codex
-args:
-  - "--dangerously-bypass-approvals-and-sandbox"
-required_env:
-  - OPENAI_API_KEY
-hooks:
-  pre_run:
-    - symlink_superpowers
-  post_run: []
-shutdown: "<<KEY:ctrl-d>>"
-idle:
-  quiescence_seconds: 5
-  ready_pattern: "^›|codex>|^>"
-startup_timeout: 60
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.codex/sessions/rollout-*.jsonl"

evals/backends/gemini-2-5-flash.yaml

@@ -1,23 +0,0 @@
-name: gemini-2-5-flash
-cli: gemini
-args:
-  - "--yolo"
-  - "-m"
-  - "gemini-2.5-flash"
-required_env: []
-hooks:
-  pre_run:
-    - link_gemini_extension
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 5
-  ready_pattern: "Type your message|^\\s*>"
-busy_pattern: "Thinking\\.\\.\\.|Executing"
-startup_timeout: 60
-turn_timeout: 300
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.gemini/tmp/*/chats/session-*.json"

evals/backends/gemini.yaml

@@ -1,23 +0,0 @@
-name: gemini
-cli: gemini
-args:
-  - "--yolo"
-  - "-m"
-  - "auto-gemini-3"
-required_env: []
-hooks:
-  pre_run:
-    - link_gemini_extension
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 5
-  ready_pattern: "Type your message|^\\s*>"
-busy_pattern: "Thinking\\.\\.\\.|Executing"
-startup_timeout: 60
-turn_timeout: 300
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.gemini/tmp/*/chats/session-*.json"

evals/bin/skill-before-tool-match

@@ -1,54 +0,0 @@
-#!/usr/bin/env bash
-# Verify a specific Skill was invoked before any Bash call whose command matches a regex.
-#
-# Usage: skill-before-tool-match <skill-name> <bash-command-regex>
-# Example: skill-before-tool-match superpowers:verification-before-completion 'git[[:space:]]+commit'
-#
-# Semantics:
-#   - If no Bash call matches the regex, PASS (vacuously — the gated event never occurred).
-#   - If Bash matches but Skill with that name never appeared earlier, FAIL.
-#   - If both appeared and Skill came first, PASS.
-#   - If Skill never appeared but Bash matched, FAIL.
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-SKILL_NAME="$1"
-BASH_REGEX="$2"
-FILE="tool_calls.jsonl"
-
-if [ ! -s "$FILE" ]; then
-    echo "FAIL: tool_calls.jsonl missing or empty"
-    exit 1
-fi
-
-# First index where Skill(skill=SKILL_NAME) appears (0-based).
-SKILL_IDX=$(
-    jq -s --arg name "$SKILL_NAME" \
-        'to_entries | map(select(.value.tool == "Skill" and (.value.args.skill // "") == $name)) | first | (.key // -1)' \
-        "$FILE"
-)
-
-# First index where Bash(command =~ BASH_REGEX) appears.
-BASH_IDX=$(
-    jq -s --arg re "$BASH_REGEX" \
-        'to_entries | map(select(.value.tool == "Bash" and ((.value.args.command // "") | test($re)))) | first | (.key // -1)' \
-        "$FILE"
-)
-
-if [ "$BASH_IDX" -lt 0 ]; then
-    echo "PASS: no Bash call matched /$BASH_REGEX/ — assertion is vacuous"
-    exit 0
-fi
-
-if [ "$SKILL_IDX" -lt 0 ]; then
-    echo "FAIL: Bash /$BASH_REGEX/ fired at line $((BASH_IDX + 1)) but Skill($SKILL_NAME) never fired"
-    exit 1
-fi
-
-if [ "$SKILL_IDX" -lt "$BASH_IDX" ]; then
-    echo "PASS: Skill($SKILL_NAME) at line $((SKILL_IDX + 1)) before Bash /$BASH_REGEX/ at line $((BASH_IDX + 1))"
-    exit 0
-else
-    echo "FAIL: Skill($SKILL_NAME) at line $((SKILL_IDX + 1)) fired after Bash /$BASH_REGEX/ at line $((BASH_IDX + 1))"
-    exit 1
-fi

evals/bin/skill-called

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Verify a specific superpowers Skill was invoked at least once.
-#
-# Usage: skill-called <skill-name>
-# Example: skill-called superpowers:systematic-debugging
-#
-# Wraps the common case of `tool-arg-match Skill '.skill == "<name>"'` so
-# scenario YAML doesn't have to embed jq quoting.
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-SKILL_NAME="$1"
-FILE="tool_calls.jsonl"
-
-if [ ! -s "$FILE" ]; then
-    echo "FAIL: tool_calls.jsonl missing or empty"
-    exit 1
-fi
-
-COUNT=$(
-    jq -s --arg name "$SKILL_NAME" \
-        '[.[] | select(.tool == "Skill" and (.args.skill // "") == $name)] | length' \
-        "$FILE"
-)
-
-if [ "$COUNT" -gt 0 ]; then
-    echo "PASS: Skill($SKILL_NAME) called $COUNT time(s)"
-    exit 0
-else
-    echo "FAIL: Skill($SKILL_NAME) never called"
-    exit 1
-fi

evals/bin/tool-arg-match

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-TOOL="$1"
-FILTER="$2"
-FILE="tool_calls.jsonl"
-
-MATCHES=$(jq -s "[.[] | select(.tool == \"$TOOL\") | select(.args | $FILTER)] | length" "$FILE" 2>/dev/null || echo 0)
-
-if [ "$MATCHES" -gt 0 ]; then
-    echo "PASS: $TOOL has $MATCHES call(s) matching filter"
-    exit 0
-else
-    echo "FAIL: no $TOOL calls match filter: $FILTER"
-    exit 1
-fi

evals/bin/tool-before

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-TOOL_A="$1"
-TOOL_B="$2"
-FILE="tool_calls.jsonl"
-
-IDX_A=$(jq -s 'to_entries | map(select(.value.tool == "'"$TOOL_A"'")) | first // empty | .key' "$FILE" 2>/dev/null)
-IDX_B=$(jq -s 'to_entries | map(select(.value.tool == "'"$TOOL_B"'")) | first // empty | .key' "$FILE" 2>/dev/null)
-
-if [ -z "$IDX_A" ] || [ "$IDX_A" = "null" ]; then
-    echo "FAIL: $TOOL_A never called"
-    exit 1
-fi
-
-if [ -z "$IDX_B" ] || [ "$IDX_B" = "null" ]; then
-    echo "FAIL: $TOOL_B never called"
-    exit 1
-fi
-
-if [ "$IDX_A" -lt "$IDX_B" ]; then
-    echo "PASS: $TOOL_A (line $((IDX_A + 1))) before $TOOL_B (line $((IDX_B + 1)))"
-    exit 0
-else
-    echo "FAIL: $TOOL_A at line $((IDX_A + 1)) occurred after $TOOL_B at line $((IDX_B + 1))"
-    exit 1
-fi

evals/bin/tool-called

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-TOOL="$1"
-FILE="tool_calls.jsonl"
-
-COUNT=$(jq -s "[.[] | select(.tool == \"$TOOL\")] | length" "$FILE" 2>/dev/null || echo 0)
-
-if [ "$COUNT" -gt 0 ]; then
-    echo "PASS: $TOOL called $COUNT time(s)"
-    exit 0
-else
-    echo "FAIL: $TOOL never called"
-    exit 1
-fi

evals/bin/tool-count

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-TOOL="$1"
-OP="$2"
-EXPECTED="$3"
-FILE="tool_calls.jsonl"
-
-COUNT=$(jq -s "[.[] | select(.tool == \"$TOOL\")] | length" "$FILE" 2>/dev/null || echo 0)
-
-case "$OP" in
-    eq)  TEST=$(( COUNT == EXPECTED )) ;;
-    gt)  TEST=$(( COUNT >  EXPECTED )) ;;
-    gte) TEST=$(( COUNT >= EXPECTED )) ;;
-    lt)  TEST=$(( COUNT <  EXPECTED )) ;;
-    lte) TEST=$(( COUNT <= EXPECTED )) ;;
-    *)   echo "Unknown operator: $OP (expected: eq, gt, gte, lt, lte)"; exit 2 ;;
-esac
-
-if [ "$TEST" -eq 1 ]; then
-    echo "PASS: $TOOL called $COUNT time(s) ($OP $EXPECTED)"
-    exit 0
-else
-    echo "FAIL: $TOOL called $COUNT time(s) (expected $OP $EXPECTED)"
-    exit 1
-fi

evals/bin/tool-match-before-tool-match

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-# Verify any Bash call with command matching a regex fires before any other Bash call
-# matching a second regex.
-#
-# Usage: tool-match-before-tool-match <tool-name> <earlier-regex> <tool-name> <later-regex>
-# Example: tool-match-before-tool-match Bash 'pytest' Bash 'git[[:space:]]+commit'
-#
-# Semantics:
-#   - If no call matches the "later" regex, PASS (vacuously — the gated event never happened).
-#   - If the "later" call fires but no "earlier" call preceded it, FAIL.
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-TOOL_A="$1"
-REGEX_A="$2"
-TOOL_B="$3"
-REGEX_B="$4"
-FILE="tool_calls.jsonl"
-
-if [ ! -s "$FILE" ]; then
-    echo "FAIL: tool_calls.jsonl missing or empty"
-    exit 1
-fi
-
-IDX_A=$(
-    jq -s --arg tool "$TOOL_A" --arg re "$REGEX_A" \
-        'to_entries | map(select(.value.tool == $tool and ((.value.args.command // "") | test($re)))) | first | (.key // -1)' \
-        "$FILE"
-)
-
-IDX_B=$(
-    jq -s --arg tool "$TOOL_B" --arg re "$REGEX_B" \
-        'to_entries | map(select(.value.tool == $tool and ((.value.args.command // "") | test($re)))) | first | (.key // -1)' \
-        "$FILE"
-)
-
-if [ "$IDX_B" -lt 0 ]; then
-    echo "PASS: no $TOOL_B call matched /$REGEX_B/ — assertion is vacuous"
-    exit 0
-fi
-
-if [ "$IDX_A" -lt 0 ]; then
-    echo "FAIL: $TOOL_B /$REGEX_B/ fired at line $((IDX_B + 1)) but no $TOOL_A /$REGEX_A/ preceded it"
-    exit 1
-fi
-
-if [ "$IDX_A" -lt "$IDX_B" ]; then
-    echo "PASS: $TOOL_A /$REGEX_A/ at line $((IDX_A + 1)) before $TOOL_B /$REGEX_B/ at line $((IDX_B + 1))"
-    exit 0
-else
-    echo "FAIL: $TOOL_A /$REGEX_A/ at line $((IDX_A + 1)) fired after $TOOL_B /$REGEX_B/ at line $((IDX_B + 1))"
-    exit 1
-fi

evals/bin/tool-not-called

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-command -v jq >/dev/null || { echo "jq required"; exit 127; }
-
-TOOL="$1"
-FILE="tool_calls.jsonl"
-
-COUNT=$(jq -s "[.[] | select(.tool == \"$TOOL\")] | length" "$FILE" 2>/dev/null || echo 0)
-
-if [ "$COUNT" -eq 0 ]; then
-    echo "PASS: $TOOL never called"
-    exit 0
-else
-    echo "FAIL: $TOOL called $COUNT time(s) (expected 0)"
-    exit 1
-fi

evals/docs/design.md

@@ -1,418 +0,0 @@
-# Drill: Superpowers Skill Compliance Benchmark
-
-**Date:** 2026-04-07
-**Ticket:** [PRI-1040](https://linear.app/prime-radiant/issue/PRI-1040)
-**Status:** Design
-
-## Thesis
-
-The value of superpowers depends on whether skills are reliably followed by *any* coding agent — not just Claude Code. Drill tests whether agents actually fire skills, follow workflows, and use native tooling when available. It is a **compliance benchmark**, not a coding ability benchmark.
-
-If a well-written skill produces consistent behavior across Claude Code and Codex, the agent-agnostic coordination layer is working. If agents diverge, Drill tells you exactly where and why.
-
-## What Drill Tests
-
-- Do agents invoke superpowers skills when they should?
-- Do they follow multi-step workflows (detect → consent → create) in the right order?
-- Do they use native tools (EnterWorktree, structured session logs) vs. raw shell commands?
-- Where do agents diverge, and what does that tell us about skill format?
-
-The first scenarios target **PRI-974 (worktree rototill)** — the area with the most cross-agent fragmentation today.
-
-## Architecture
-
-Three layers, each with a single responsibility:
-
-```
-┌─────────────────────────────────────────┐
-│  CLI  (click)                           │
-│  run / compare / list                   │
-├─────────────────────────────────────────┤
-│  Engine                                 │
-│  ┌───────────┐ ┌───────┐ ┌──────────┐  │
-│  │ Session   │ │ Actor │ │ Verifier │  │
-│  │ (tmux)    │ │ (LLM) │ │ (LLM)   │  │
-│  └───────────┘ └───────┘ └──────────┘  │
-├─────────────────────────────────────────┤
-│  Backends                               │
-│  claude / codex / (future: gemini)      │
-├─────────────────────────────────────────┤
-│  Setup                                  │
-│  template repo + helpers + assertions   │
-└─────────────────────────────────────────┘
-```
-
-- **CLI** — `drill run <scenario> --backend claude`, `drill compare <scenario>`, `drill list`
-- **Engine** — Orchestrates the full run lifecycle (setup → session → actor loop → collect → verify → results)
-- **Session** — tmux lifecycle: create session, send-keys, capture-pane, kill session
-- **Actor** — Sonnet with rolling context. Gets all scenario intents as a goal stack + terminal screens. Outputs what to type next, or `<<DONE>>`/`<<STUCK>>`.
-- **Verifier** — Sonnet (near-zero temperature) with full session log + filesystem state + tool call log + criteria list. Returns per-criterion pass/fail with cited evidence + freeform observations.
-- **Backends** — Each backend knows: CLI command, auto-approve flags, plugin loading, idle detection, shutdown command, session log location.
-- **Setup** — Clone template repo → run backend pre_run hooks → run scenario helpers → run setup assertions → fail fast if invariants violated.
-
-## Engine Flow
-
-```
-1. LOAD
-   - Parse scenario YAML
-   - Parse backend YAML
-   - Validate required env vars (fail fast)
-
-2. SETUP
-   - Clone template repo to temp dir
-   - Run backend pre_run hooks (codex symlink, etc.)
-   - Run scenario setup helpers
-   - Run setup assertions → abort if any fail
-
-3. SESSION
-   - Create tmux session (backend-specific terminal dimensions)
-   - Launch agent CLI in tmux pane
-   - Wait for startup ready pattern
-
-4. ACTOR LOOP
-   - For each turn (up to max_turns):
-     a. Wait for idle (quiescence + ready pattern)
-     b. Capture terminal pane → append to rolling context
-     c. Send to Actor LLM: system prompt + rolling context + ALL intents + user_posture
-     d. Actor responds with text to type, <<DONE>>, or <<STUCK>>
-     e. If <<DONE>> or <<STUCK>> → break
-     f. Send keystrokes via tmux send-keys
-     g. Per-turn timeout → <<STUCK>> if exceeded
-   - Special keys via <<KEY:name>> convention (e.g., <<KEY:ctrl-c>>)
-
-5. COLLECT
-   - Capture final terminal state
-   - Send shutdown command (backend-specific: /exit, Ctrl-D, etc.)
-   - Wait for process exit (with timeout)
-   - Snapshot filesystem (file tree, git state, worktree list)
-   - Collect backend session logs → tool_calls.jsonl
-   - Kill tmux session (cleanup if process didn't exit cleanly)
-
-6. VERIFY
-   - Send to Verifier LLM: session.log + filesystem.json + tool_calls.jsonl + criteria
-   - Verifier receives criteria but NOT actor intents (reduces confirmation bias)
-   - Verifier returns per-criterion pass/fail with evidence + rationale + observations
-   - Output as structured JSON (verdict.json)
-
-7. RESULTS
-   - Write to results/<scenario>/<backend>/<timestamp>/
-   - Print summary to stdout
-```
-
-## Backend Abstraction
-
-Each backend is a YAML config. Backends own: CLI invocation, idle detection, shutdown, session log collection, and pre/post-run hooks.
-
-```yaml
-# backends/claude.yaml
-name: claude
-cli: claude
-args:
-  - "--dangerously-skip-permissions"
-  - "--plugin-dir"
-  - "${SUPERPOWERS_ROOT}"
-required_env:
-  - ANTHROPIC_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run: []    # no repo setup needed; plugin loaded via --plugin-dir
-  post_run: []
-shutdown: "/exit"
-idle:
-  quiescence_seconds: 3
-  ready_pattern: "^❯|^\\$|Human:"
-startup_timeout: 30
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.claude/projects/**/session-*.jsonl"
-  match_by: timestamp
-```
-
-```yaml
-# backends/codex.yaml
-name: codex
-cli: codex
-args:
-  - "--dangerously-bypass-approvals-and-sandbox"
-required_env:
-  - OPENAI_API_KEY
-  - SUPERPOWERS_ROOT
-hooks:
-  pre_run:
-    - symlink_superpowers   # creates .agents/skills/superpowers symlink in test repo
-  post_run: []
-shutdown: "<<KEY:ctrl-d>>"
-idle:
-  quiescence_seconds: 5
-  ready_pattern: "codex>|^>"
-startup_timeout: 30
-terminal:
-  cols: 200
-  rows: 50
-session_logs:
-  pattern: "~/.codex/sessions/rollout-*.jsonl"
-  match_by: timestamp
-```
-
-New backends = new YAML file. Backend variants (e.g., `codex-workspace-write.yaml`) are just copies with different args — no inheritance system needed. Scenarios reference backends by name.
-
-## Scenario Format
-
-Scenarios are YAML. They describe *what* to test, not *how* each backend works.
-
-```yaml
-scenario: worktree-creation-from-main
-description: "Agent creates an isolated worktree from main branch"
-user_posture: naive   # or spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-    - "git worktree list | wc -l | grep 1"
-
-turns:
-  - intent: >
-      Ask the agent to create an isolated workspace
-      for building a login feature.
-  - intent: "Confirm consent if the agent asks."
-
-limits:
-  max_turns: 20
-  turn_timeout: 120   # seconds per turn
-
-verify:
-  criteria:
-    - "Agent detected it was on main, not in an existing worktree"
-    - "Agent asked for consent before creating the worktree"
-    - "A worktree or isolated workspace now exists with a feature branch"
-    - "Agent used the most appropriate tool available for its platform to create the worktree"
-  observe: true   # verifier can add freeform observations
-```
-
-### User Posture
-
-Each scenario has a `user_posture` field:
-
-- **naive** — User describes what they want in plain language. Tests whether the agent's superpowers skills fire without hand-holding.
-- **spec-aware** — User references specific skills or conventions by name. Tests whether the agent follows the spec when pointed at it.
-
-The delta between naive and spec-aware results for the same scenario is the most interesting product signal. A small delta means strong conveyance. A large delta means the skill format needs work.
-
-### Turn Intents
-
-Intents are a **priority-ordered goal stack**, not a rigid script. The actor receives all intents and decides which one applies to the current terminal state. Some intents are conditional ("Confirm consent if the agent asks") and may never fire.
-
-## Setup
-
-### Template Repo
-
-A real git repo checked into `fixtures/template-repo/`. Cloned to a temp directory per run. Covers the 80% common case.
-
-Contents:
-- `package.json` — minimal Node project metadata (name, version)
-- `src/index.js` — simple entry point (~10 lines)
-- `src/utils.js` — helper module (~10 lines)
-- `README.md` — basic project description
-- 3-4 commits on `main` with realistic messages (e.g., "initial commit", "add utils module", "update readme")
-- No existing worktrees, branches, or tags beyond `main`
-
-This is intentionally minimal — just enough for agents to recognize it as a real project. Scenario-specific state (extra branches, worktrees, detached HEAD) is added by setup helpers.
-
-### Setup Helpers
-
-Python functions in `setup_helpers/` that modify the cloned repo for specific scenarios:
-
-- `create_base_repo(workdir)` — Clone template, verify structure
-- `add_worktree(workdir, branch, path)` — Create an existing worktree (for "already inside" scenarios)
-- `detach_head(workdir)` — Simulate Codex App detached HEAD state
-- `symlink_superpowers(workdir)` — Create `.agents/skills/superpowers` symlink (codex pre_run hook)
-
-### Setup Assertions
-
-Run after all setup completes, before the agent launches. If any fail, the scenario aborts with a clear "setup invariant violated" error — not a mysterious agent failure 10 turns later.
-
-## Plugin Loading
-
-Each backend loads superpowers differently. The harness manages this per-run with no global config mutation:
-
-| Backend | Mechanism | Harness action |
-|---------|-----------|----------------|
-| Claude Code | `--plugin-dir` CLI flag | Pass flag pointing at superpowers checkout |
-| Codex | `.agents/skills/` in repo | Backend pre_run hook creates symlink |
-
-This means Drill can test draft skill changes by pointing at a branch checkout of superpowers.
-
-## Post-Session Tool Call Collection
-
-Both backends write structured session logs that record every tool invocation:
-
-| Backend | Log location | Format |
-|---------|-------------|--------|
-| Claude Code | `~/.claude/projects/**/session-*.jsonl` | JSONL with tool names + args |
-| Codex | `~/.codex/sessions/rollout-*.jsonl` | JSONL with `LocalShellCall`, `FunctionCall`, etc. |
-
-The harness snapshots each backend's log directory before the session starts. After shutdown, it diffs the directory to find only files created during the run — no timestamp matching needed, no cross-contamination from concurrent sessions or prior runs.
-
-Collected logs are normalized into a common `tool_calls.jsonl` format before the verifier sees them:
-
-```json
-{"tool": "EnterWorktree", "args": {"branch": "add-login"}, "source": "native"}
-{"tool": "Bash", "args": {"command": "git worktree add ..."}, "source": "shell"}
-```
-
-Each backend defines a normalizer function that maps its native log format (Claude Code's tool call entries, Codex's `ResponseItem` records) into this common schema. The verifier never sees raw backend-specific logs.
-
-## Actor & Verifier LLM Design
-
-### Actor
-
-- **Model:** Sonnet
-- **Temperature:** 0.7 (realistic user variation)
-- **Context:** Rolling (full conversation history). Sessions are short enough (~5-20 turns) that token cost is not a concern.
-- **Input:** System prompt + rolling terminal captures + all intents + user_posture
-- **Output:** Structured JSON via Anthropic SDK tool_use: `{"action": "type", "text": "..."}`, `{"action": "done"}`, `{"action": "stuck"}`, or `{"action": "key", "key": "ctrl-c"}`. The harness parses this and sends keystrokes — no free-text sanitization needed.
-- **Prompt:** Versioned template at `prompts/actor.md`
-
-### Verifier
-
-- **Model:** Sonnet
-- **Temperature:** Near-zero (deterministic judgment)
-- **Input:** session.log + filesystem.json + tool_calls.jsonl + criteria list. Does NOT receive actor intents or scenario narrative (reduces confirmation bias).
-- **Output:** Structured JSON with per-criterion verdict/evidence/rationale + observations
-- **Prompt:** Versioned template at `prompts/verifier.md`
-
-## Results & Compare
-
-### Results Structure
-
-```
-results/
-  <scenario>/
-    <backend>/
-      <timestamp>/
-        session.log        # raw tmux capture
-        filesystem.json    # post-run git/file state snapshot
-        tool_calls.jsonl   # collected from backend session logs
-        verdict.json       # verifier output
-        meta.json          # run metadata (backend, duration, turns, model versions)
-```
-
-### Compare Command
-
-`drill compare` reads existing results from prior `drill run` invocations. It does not run backends itself — run each backend separately first, then compare.
-
-```
-$ drill run worktree-creation-from-main --backend claude
-$ drill run worktree-creation-from-main --backend codex
-$ drill compare worktree-creation-from-main
-
-Scenario: worktree-creation-from-main (naive posture)
-
-Summary:
-┌──────────┬────────┬───────┬───────┐
-│ Backend  │ Result │ Score │ Turns │
-├──────────┼────────┼───────┼───────┤
-│ claude   │ PASS   │ 4/4   │ 6     │
-│ codex    │ FAIL   │ 2/4   │ 12    │
-└──────────┴────────┴───────┴───────┘
-
-Detail:
-┌────────────────────────────────┬────────┬────────┐
-│ Criterion                      │ claude │ codex  │
-├────────────────────────────────┼────────┼────────┤
-│ Detected on main               │ ✓      │ ✓      │
-│ Asked consent                  │ ✓      │ ✗      │
-│ Worktree exists                │ ✓      │ ✓      │
-│ Used native tools              │ ✓      │ ✗      │
-└────────────────────────────────┴────────┴────────┘
-
-Observations:
-  claude: "Agent cited the using-git-worktrees skill by name"
-  codex:  "Agent created worktree but skipped consent step entirely"
-```
-
-## Project Structure
-
-```
-drill/
-├── drill/
-│   ├── __init__.py
-│   ├── cli.py              # click CLI: run, compare, list
-│   ├── engine.py            # orchestrates the full run lifecycle
-│   ├── session.py           # tmux session management
-│   ├── actor.py             # actor LLM calls
-│   ├── verifier.py          # verifier LLM calls
-│   ├── setup.py             # template repo cloning, helpers, assertions
-│   └── backend.py           # loads backend YAML, builds commands
-├── backends/
-│   ├── claude.yaml
-│   └── codex.yaml
-├── prompts/
-│   ├── actor.md
-│   └── verifier.md
-├── scenarios/
-│   ├── worktree-creation-from-main.yaml
-│   ├── worktree-already-inside.yaml
-│   ├── worktree-codex-detached-head.yaml
-│   └── worktree-consent-flow.yaml
-├── fixtures/
-│   └── template-repo/       # base git repo, cloned per run
-├── setup_helpers/
-│   ├── __init__.py
-│   ├── base.py              # create_base_repo, common git ops
-│   └── worktree.py          # add_worktree, detach_head, etc.
-├── results/                  # gitignored, populated by runs
-├── pyproject.toml             # package metadata + [project.scripts] entry point
-└── README.md
-```
-
-## Phase 1 Scope
-
-- Claude Code + Codex backends
-- 4 PRI-974 worktree scenarios (creation, already-inside, detached-head, consent)
-- Both user postures (naive + spec-aware) per scenario
-- Template repo + setup helpers + assertions
-- Actor + verifier with prompts
-- `drill run` and `drill compare` commands
-- Results storage
-
-## Phase 2 (Future)
-
-- Gemini CLI backend
-- Backend variants (e.g., `codex-workspace-write.yaml` for sandbox mode testing)
-- Verifier flakiness mitigation (3x voting, agreement tracking)
-- Cost tracking and token usage reporting
-- Docker isolation for reproducibility
-- CI integration
-- Scenarios beyond worktrees (stacked PRs, git-spice, brainstorming)
-
-## Installation
-
-```bash
-pip install -e .    # installs 'drill' console script
-```
-
-Requires `tmux` installed as a system dependency.
-
-## Dependencies
-
-- Python 3.11+
-- `click` — CLI framework
-- `pyyaml` — scenario and backend config parsing
-- `anthropic` — Anthropic Python SDK for actor/verifier LLM calls (structured tool_use output)
-- `jinja2` — prompt template rendering
-- `pydantic` — verdict schema validation (retry on malformed verifier output)
-- `tmux` — session driving (system dependency)
-
-## Non-Goals
-
-- Not a coding ability benchmark (SWE-bench covers that)
-- Not an LLM evaluation framework (promptfoo covers that)
-- Not a generic terminal automation tool (Terminal-Bench covers that)
-- No CI in phase 1
-- No Docker in phase 1

evals/docs/manual-testing.md

@@ -1,93 +0,0 @@
-# Manual Testing (Codex App)
-
-Some scenarios cannot run automatically because drill has no harness adapter for the target — the Codex App desktop client has no CLI or tmux entry point the way `claude` and `codex` do. These scenarios are marked `manual: true` in their YAML and use a human-in-the-loop protocol.
-
-## Protocol
-
-Three phases. The agent never runs Codex App directly. The tester never writes a verdict by hand.
-
-1. **Agent prepares the handoff** — reads the scenario file, renders setup + turn intents into something a human can act on, hands the package to the tester.
-2. **Tester executes** — sets up the repo fixture, opens Codex App, pastes the prompt, handles any follow-ups, copies the transcript + final filesystem state back to the agent.
-3. **Agent judges and records** — evaluates the transcript against `verify.criteria`, writes a verdict JSON, saves to `results/<scenario>/codex-app/YYYY-MM-DD-manual/verdict.json`.
-
-## Phase 1: Agent prepares the handoff
-
-Deliver as one self-contained message to the tester:
-
-### Fixture state
-Exact repo state Codex App should be launched against. Pull from `setup.notes` if present, otherwise translate `setup.helpers` + `setup.assertions` into prose. Include: which repo/directory, branch, whether to expect a worktree vs normal checkout, any required/forbidden files (e.g. `.gitignore` entries).
-
-### Prompt to paste
-Render turn 1's `intent` as a natural first-person message the tester can paste verbatim into Codex App. **Don't leak internal test language** like *"Do NOT say 'create a worktree'"* — that's instruction for the test author, not the end user. Convert it to what a real user would actually type.
-
-Example:
-> Intent: *"Ask the agent to use the worktree skill to get set up for a notifications feature. Do NOT say 'create a worktree' — just reference the skill by name."*
->
-> Rendered prompt: *"hey, can you use the worktree skill to get me set up for a notifications feature?"*
-
-### Follow-up guidance
-For each additional turn, give the tester a short decision rule — not a verbatim script. E.g. *"If the agent asks a clarifying question like branch name, answer concisely. If it stops to ask whether you want a worktree at all, tell it you already asked for the skill and it should proceed."*
-
-### What to capture
-Ask the tester to paste back:
-- Full agent transcript (messages, tool calls, tool outputs)
-- Final filesystem state if criteria depend on it (`git worktree list`, directory tree, branch state)
-- Any observations they want on the record
-
-## Phase 2: Tester executes
-
-1. Set up the repo fixture per the instructions
-2. Open Codex App in that repo
-3. Paste the prompt
-4. Follow up per the guidance
-5. Copy the transcript + filesystem state back to the agent
-
-## Phase 3: Agent judges and records
-
-For each criterion in `verify.criteria`, write one entry:
-
-```json
-{
-  "criterion": "<verbatim from scenario>",
-  "passed": true | false,
-  "evidence": "<quoted snippet from transcript>",
-  "rationale": "<only if passed is inconclusive or needs context>"
-}
-```
-
-**Rules:**
-- Quote the transcript directly in `evidence`. No paraphrasing.
-- If a criterion is genuinely inconclusive from the transcript, mark `passed: false` with `rationale` explaining what was missing. Don't guess.
-- Don't grade on intent you can't see. The agent's internal thoughts aren't visible — only messages, tool calls, and results.
-
-### Verdict file
-
-Save to `results/<scenario>/codex-app/YYYY-MM-DD-manual/verdict.json`:
-
-```json
-{
-  "scenario": "<scenario-name>",
-  "backend": "codex-app",
-  "manual": true,
-  "user_posture": "<spec-aware|naive|...>",
-  "passed": <true iff every criterion.passed is true>,
-  "criteria": [ ... ],
-  "notes": "<optional: cross-criterion observations>"
-}
-```
-
-Matches the format of the existing `results/worktree-codex-app-detached-head/codex-app/2026-04-09-manual/verdict.json`.
-
-## When to invoke
-
-- A scenario's YAML has `manual: true`
-- The tester explicitly asks for a manual Codex App run of any scenario
-- An automated test result is inconclusive and we want a human-verified cross-check
-
-Do NOT use this procedure for scenarios drill can run itself (`claude`, `codex`, `gemini` backends) — use `drill run` instead.
-
-## Pitfalls
-
-- **Don't skip the fixture step.** Codex App's default environment (detached HEAD under `$CODEX_HOME/worktrees/`) is load-bearing for worktree scenarios. The same prompt gives different results in a normal checkout.
-- **Don't render prompts literally.** Scenario intents are written for test authors; they often contain "Do NOT mention X" style instructions. Translate before handing to the tester.
-- **Don't grade on missing evidence.** If the transcript doesn't show the agent doing something the criterion asks about, that's a fail, not a pass-by-default.

evals/docs/pressure-and-red-testing.md

@@ -1,89 +0,0 @@
-# Pressure / RED phase testing in drill
-
-## What "RED phase" means
-
-The bash test family in superpowers/tests/ used three implicit phases
-when stress-testing skill content:
-
-* **GREEN** — current skill text. Baseline behavior under normal user
-  prompts. This is what most drill scenarios exercise.
-* **PRESSURE** — current skill text, but the user prompt creates
-  conditions that make the skill's recommended path inconvenient
-  (urgency, an "easier" alternative already on disk, etc.). Lifted
-  as `worktree-creation-under-pressure.yaml`.
-* **RED** — *modified* skill text where the section under test has
-  been removed or weakened. Used to confirm a passing GREEN/PRESSURE
-  result actually depended on the skill text and isn't just baseline
-  model behavior.
-
-GREEN and PRESSURE both run against the current `SUPERPOWERS_ROOT`.
-RED needs a *different* superpowers checkout — one with the section
-under test stripped out — and runs the same scenario against that.
-
-## The drill primitive: vary `SUPERPOWERS_ROOT`
-
-Every backend YAML interpolates `${SUPERPOWERS_ROOT}` into its
-`--plugin-dir` arg (claude.yaml line 6, gemini.yaml line 5, etc.).
-That env var is the only knob you need: point drill at a different
-plugin checkout and the agent under test loads a different version
-of the skill.
-
-```bash
-# GREEN: current skill text
-drill run worktree-creation-from-main -b claude
-
-# RED: same scenario, against a checkout where Step 1a is deleted
-SUPERPOWERS_ROOT=/path/to/superpowers-without-step-1a \
-  drill run worktree-creation-from-main -b claude
-```
-
-Compare verdicts. If GREEN passes and RED fails, the skill text is
-load-bearing. If both pass, the model produces the right behavior
-without the skill — meaning either the skill is redundant or the
-test isn't probing what it claims to probe.
-
-## Recommended workflow
-
-1. Make a git worktree of superpowers at the commit/branch you want
-   to test. For RED variants, edit the skill in that worktree to
-   remove the section under test.
-
-   ```bash
-   cd ~/Documents/GitHub/superpowers/superpowers
-   git worktree add ../superpowers-red-no-step-1a HEAD
-   # edit skills/using-git-worktrees/SKILL.md in the worktree
-   ```
-
-2. Run the same drill scenario against each variant. Use
-   `--n N` to get statistical signal — single runs are noisy,
-   especially under pressure conditions.
-
-   ```bash
-   for variant in main red-no-step-1a; do
-     SUPERPOWERS_ROOT=~/Documents/GitHub/superpowers/superpowers-${variant#main}superpowers \
-       drill run worktree-creation-from-main -b claude --n 10
-   done
-   ```
-
-3. Compare with `drill compare`. Look for the RED variant's pass
-   rate dropping (skill is load-bearing) or holding (skill is
-   redundant or scenario isn't probing what it claims).
-
-## When to add a new pressure scenario vs. add a turn variation
-
-* **New scenario** when the *filesystem* setup is different (e.g.,
-  pre-existing `.worktrees/` for the worktree-pressure case).
-  Setup helpers are scenario-scoped.
-* **New `--n` sweep with different prompts** when only the
-  *user prompt* shape varies (e.g., urgency, framing).
-
-Drill doesn't yet have a way to vary turn intents within a single
-scenario YAML — multi-prompt sweeps require multiple scenario files
-or running the same scenario with different intents externally.
-
-## Open follow-ups
-
-* `--plugins=A,B,C` sweep dimension (parallel to `--models`) so a
-  single drill invocation can run RED + GREEN + PRESSURE variants
-  in one batch and `drill compare` shows them side-by-side. Not yet
-  implemented; tracked as drill-internal future work.

evals/drill/__init__.py

@@ -1,3 +0,0 @@
-"""Drill: Superpowers skill compliance benchmark."""
-
-__version__: str = "0.1.0"

evals/drill/__main__.py

@@ -1,5 +0,0 @@
-"""Allow running drill as `python3 -m drill`."""
-
-from drill.cli import main
-
-main()

evals/drill/actor.py

@@ -1,81 +0,0 @@
-"""Actor LLM: simulates a user driving an agent session."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-import anthropic
-from jinja2 import Template
-
-ACTOR_TOOL: dict[str, Any] = {
-    "name": "terminal_action",
-    "description": "Send an action to the terminal session.",
-    "input_schema": {
-        "type": "object",
-        "properties": {
-            "action": {
-                "type": "string",
-                "enum": ["type", "done", "stuck", "key"],
-                "description": "The action to take.",
-            },
-            "text": {
-                "type": "string",
-                "description": "Text to type (only for 'type' action).",
-            },
-            "key": {
-                "type": "string",
-                "description": "Special key to send (only for 'key' action, e.g., 'ctrl-c').",
-            },
-        },
-        "required": ["action"],
-    },
-}
-
-
-@dataclass
-class ActorAction:
-    action: str
-    text: str | None = None
-    key: str | None = None
-
-    @classmethod
-    def from_tool_result(cls, data: dict[str, Any]) -> ActorAction:
-        return cls(action=data["action"], text=data.get("text"), key=data.get("key"))
-
-
-class Actor:
-    def __init__(self, model: str = "claude-sonnet-4-6", temperature: float = 0.7) -> None:
-        self.model = model
-        self.temperature = temperature
-        self.captures: list[str] = []
-        self._system_prompt: str = ""
-        self._client: anthropic.Anthropic = anthropic.Anthropic()
-
-    def build_system_prompt(self, posture: str, intents: list[str]) -> str:
-        template_path = Path(__file__).parent.parent / "prompts" / "actor.md"
-        template = Template(template_path.read_text())
-        self._system_prompt = template.render(posture=posture, intents=intents)
-        return self._system_prompt
-
-    def append_capture(self, terminal_output: str) -> None:
-        self.captures.append(terminal_output)
-
-    def build_messages(self) -> list[dict[str, str]]:
-        return [{"role": "user", "content": capture} for capture in self.captures]
-
-    def decide(self) -> ActorAction:
-        response = self._client.messages.create(
-            model=self.model,
-            max_tokens=1024,
-            temperature=self.temperature,
-            system=self._system_prompt,
-            tools=[ACTOR_TOOL],  # ty: ignore[invalid-argument-type]
-            tool_choice={"type": "tool", "name": "terminal_action"},
-            messages=self.build_messages(),  # ty: ignore[invalid-argument-type]
-        )
-        for block in response.content:
-            if block.type == "tool_use":
-                return ActorAction.from_tool_result(block.input)
-        raise RuntimeError("Actor did not return a tool_use block")

evals/drill/assertions.py

@@ -1,89 +0,0 @@
-"""Post-session deterministic assertions for drill scenarios."""
-
-from __future__ import annotations
-
-import os
-import subprocess
-from dataclasses import dataclass
-from pathlib import Path
-
-from drill.verifier import CriterionResult
-
-
-@dataclass
-class AssertionResult:
-    command: str
-    passed: bool
-    exit_code: int
-    stdout: str
-    stderr: str
-
-    def to_criterion_result(self) -> CriterionResult:
-        evidence = f"exit code {self.exit_code}"
-        if self.stdout:
-            evidence += f"\nstdout: {self.stdout}"
-        if self.stderr:
-            evidence += f"\nstderr: {self.stderr}"
-        return CriterionResult(
-            criterion=f"[assertion] {self.command}",
-            verdict="pass" if self.passed else "fail",
-            evidence=evidence,
-            rationale="Deterministic assertion " + ("passed" if self.passed else "failed"),
-            source="assertion",
-        )
-
-
-def run_verify_assertions(
-    assertions: list[str],
-    results_dir: Path,
-    workdir: Path,
-    *,
-    timeout_seconds: int = 10,
-) -> list[AssertionResult]:
-    bin_dir = Path(__file__).parent.parent / "bin"
-    env = {
-        **os.environ,
-        "DRILL_WORKDIR": str(workdir),
-        "PATH": f"{bin_dir}:{os.environ.get('PATH', '')}",
-    }
-    results: list[AssertionResult] = []
-    for cmd in assertions:
-        try:
-            proc = subprocess.run(
-                ["bash", "-c", cmd],
-                cwd=results_dir,
-                capture_output=True,
-                text=True,
-                env=env,
-                timeout=timeout_seconds,
-            )
-            results.append(
-                AssertionResult(
-                    command=cmd,
-                    passed=proc.returncode == 0,
-                    exit_code=proc.returncode,
-                    stdout=proc.stdout.strip(),
-                    stderr=proc.stderr.strip(),
-                )
-            )
-        except subprocess.TimeoutExpired:
-            results.append(
-                AssertionResult(
-                    command=cmd,
-                    passed=False,
-                    exit_code=124,
-                    stdout="",
-                    stderr=f"Timed out after {timeout_seconds}s",
-                )
-            )
-        except Exception as e:
-            results.append(
-                AssertionResult(
-                    command=cmd,
-                    passed=False,
-                    exit_code=-1,
-                    stdout="",
-                    stderr=str(e),
-                )
-            )
-    return results

evals/drill/backend.py

@@ -1,111 +0,0 @@
-"""Backend config loader and command builder."""
-
-from __future__ import annotations
-
-import os
-import re
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-
-@dataclass
-class Backend:
-    name: str
-    cli: str
-    args: list[str]
-    required_env: list[str]
-    hooks: dict[str, list[str]]
-    shutdown: str
-    idle: dict[str, Any]
-    startup_timeout: int
-    terminal: dict[str, int]
-    session_logs: dict[str, str]
-    turn_timeout: int | None = None
-    busy_pattern: str = ""
-    max_busy_seconds: int = 1800
-
-    def build_command(self, workdir: str) -> list[str]:
-        resolved = [_interpolate_env(arg) for arg in self.args]
-        return [self.cli, *resolved]
-
-    def validate_env(self) -> None:
-        missing = [v for v in self.required_env if not os.environ.get(v)]
-        if missing:
-            raise OSError(
-                f"Missing required environment variables for {self.name} backend: "
-                + ", ".join(missing)
-            )
-
-    def is_ready_line(self, line: str) -> bool:
-        pattern = self.idle.get("ready_pattern", "")
-        return bool(re.search(pattern, line))
-
-    def is_busy_line(self, line: str) -> bool:
-        if not self.busy_pattern:
-            return False
-        return bool(re.search(self.busy_pattern, line))
-
-    @property
-    def quiescence_seconds(self) -> float:
-        return self.idle.get("quiescence_seconds", 5)
-
-    @property
-    def cols(self) -> int:
-        return self.terminal.get("cols", 200)
-
-    @property
-    def rows(self) -> int:
-        return self.terminal.get("rows", 50)
-
-    @property
-    def model(self) -> str | None:
-        """Model name from args (looks for --model or -m flag)."""
-        for i, arg in enumerate(self.args):
-            if arg in ("--model", "-m") and i + 1 < len(self.args):
-                return self.args[i + 1]
-        return None
-
-    @property
-    def family(self) -> str:
-        """Normalize backend name to a family for log-dir / normalizer dispatch."""
-        for fam in ("claude", "codex", "gemini"):
-            if self.name == fam or self.name.startswith(f"{fam}-"):
-                return fam
-        return "other"
-
-
-def load_backend(name: str, backends_dir: Path) -> Backend:
-    path = backends_dir / f"{name}.yaml"
-    if not path.exists():
-        raise FileNotFoundError(f"Backend config not found: {path}")
-    with open(path) as f:
-        data = yaml.safe_load(f)
-    return Backend(
-        name=data["name"],
-        cli=data["cli"],
-        args=data.get("args", []),
-        required_env=data.get("required_env", []),
-        hooks=data.get("hooks", {"pre_run": [], "post_run": []}),
-        shutdown=data.get("shutdown", "/exit"),
-        idle=data.get("idle", {}),
-        startup_timeout=data.get("startup_timeout", 30),
-        terminal=data.get("terminal", {"cols": 200, "rows": 50}),
-        session_logs=data.get("session_logs", {}),
-        turn_timeout=data.get("turn_timeout"),
-        busy_pattern=data.get("busy_pattern", ""),
-        max_busy_seconds=data.get("max_busy_seconds", 1800),
-    )
-
-
-def _interpolate_env(value: str) -> str:
-    def replacer(match: re.Match[str]) -> str:
-        var = match.group(1)
-        val = os.environ.get(var)
-        if val is None:
-            raise OSError(f"Environment variable {var} not set")
-        return val
-
-    return re.sub(r"\$\{(\w+)\}", replacer, value)

evals/drill/cli.py

@@ -1,154 +0,0 @@
-"""Drill CLI: run, compare, list."""
-
-from __future__ import annotations
-
-import os
-import secrets
-from pathlib import Path
-
-import click
-from dotenv import load_dotenv
-
-PROJECT_ROOT: Path = Path(__file__).parent.parent
-
-load_dotenv(PROJECT_ROOT / ".env")
-
-
-def _set_superpowers_root_default() -> None:
-    """Default SUPERPOWERS_ROOT to the parent of evals/ if not already set.
-
-    Drill historically required contributors to export SUPERPOWERS_ROOT
-    pointing at the superpowers checkout. After lifting drill into
-    superpowers/evals/, the parent of PROJECT_ROOT is always the
-    superpowers root, so we can supply this default automatically.
-
-    Existing SUPERPOWERS_ROOT environment values are respected as overrides.
-    """
-    os.environ.setdefault("SUPERPOWERS_ROOT", str(PROJECT_ROOT.parent))
-
-
-_set_superpowers_root_default()
-
-
-@click.group()
-def main() -> None:
-    """Drill: Superpowers skill compliance benchmark."""
-    pass
-
-
-@main.command()
-@click.argument("scenario")
-@click.option("--backend", "-b", default=None, help="Backend name (e.g., claude, codex)")
-@click.option("--models", "-m", default=None, help="Comma-separated backend names for sweep")
-@click.option("--n", "n_runs", type=int, default=1, help="Number of repetitions per backend")
-@click.option(
-    "--backends-dir",
-    type=click.Path(exists=True, path_type=Path),
-    default=PROJECT_ROOT / "backends",
-)
-@click.option(
-    "--scenarios-dir",
-    type=click.Path(exists=True, path_type=Path),
-    default=PROJECT_ROOT / "scenarios",
-)
-@click.option(
-    "--fixtures-dir",
-    type=click.Path(exists=True, path_type=Path),
-    default=PROJECT_ROOT / "fixtures",
-)
-@click.option("--results-dir", type=click.Path(path_type=Path), default=PROJECT_ROOT / "results")
-def run(
-    scenario: str,
-    backend: str | None,
-    models: str | None,
-    n_runs: int,
-    backends_dir: Path,
-    scenarios_dir: Path,
-    fixtures_dir: Path,
-    results_dir: Path,
-) -> None:
-    """Run a scenario against one or more backends."""
-    if n_runs < 1:
-        raise click.ClickException("--n must be at least 1")
-
-    if models:
-        backend_names = [b.strip() for b in models.split(",") if b.strip()]
-    elif backend:
-        backend_names = [backend]
-    else:
-        raise click.ClickException("Either --backend or --models is required")
-
-    scenario_path = scenarios_dir / f"{scenario}.yaml"
-    if not scenario_path.exists():
-        raise click.ClickException(f"Scenario not found: {scenario_path}")
-
-    sweep_id = secrets.token_hex(4)
-
-    from drill.sweep import Sweep
-
-    sweep = Sweep(
-        scenario_path=scenario_path,
-        backend_names=backend_names,
-        backends_dir=backends_dir,
-        fixtures_dir=fixtures_dir,
-        results_dir=results_dir,
-        n=n_runs,
-        sweep_id=sweep_id,
-    )
-
-    total = len(backend_names) * n_runs
-    click.echo(
-        f"Running {scenario} | backends: {', '.join(backend_names)} | "
-        f"n={n_runs} | total runs: {total} | sweep: {sweep_id}"
-    )
-
-    groups = sweep.run_all()
-
-    for group in groups:
-        passed = sum(1 for r in group.runs if r.status == "pass")
-        failed = sum(1 for r in group.runs if r.status == "fail")
-        errored = sum(1 for r in group.runs if r.status == "error")
-        click.echo(f"\n{group.backend}: {passed} passed, {failed} failed, {errored} errors")
-        if group.partial:
-            click.echo("  (interrupted — partial results)")
-
-
-@main.command("list")
-@click.option(
-    "--scenarios-dir",
-    type=click.Path(exists=True, path_type=Path),
-    default=PROJECT_ROOT / "scenarios",
-)
-def list_scenarios(scenarios_dir: Path) -> None:
-    """List available scenarios."""
-    import yaml
-
-    for f in sorted(scenarios_dir.glob("*.yaml")):
-        with open(f) as fh:
-            data = yaml.safe_load(fh)
-        name = data.get("scenario", f.stem)
-        desc = data.get("description", "")
-        click.echo(f"  {name:40s} {desc}")
-
-
-@main.command()
-@click.argument("scenario")
-@click.option("--sweep", "sweep_id", default=None, help="Filter by sweep ID")
-@click.option(
-    "--results-dir",
-    type=click.Path(exists=True, path_type=Path),
-    default=PROJECT_ROOT / "results",
-)
-def compare(scenario: str, sweep_id: str | None, results_dir: Path) -> None:
-    """Compare results across backends for a scenario."""
-    from drill.compare import format_compare_output, load_scenario_results
-
-    scenario_dir = results_dir / scenario
-    if not scenario_dir.exists():
-        raise click.ClickException(f"No results found for: {scenario}")
-
-    results = load_scenario_results(scenario_dir, sweep_id=sweep_id)
-    if not results:
-        raise click.ClickException(f"No results found for: {scenario}")
-
-    click.echo(format_compare_output(scenario, results))

evals/drill/compare.py

@@ -1,255 +0,0 @@
-"""Compare: load and aggregate drill results across backends and runs."""
-
-from __future__ import annotations
-
-import json
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-from drill.stats import wilson_ci
-from drill.verifier import Verdict
-
-
-@dataclass
-class BackendResult:
-    backend: str
-    total_runs: int
-    passed_runs: int
-    errored_runs: int
-    avg_turns: float
-    criterion_counts: dict[str, tuple[int, int]]  # criterion -> (passed, total)
-    sweep_id: str | None
-    timestamp: str | None
-    partial: bool
-
-    @property
-    def pass_rate(self) -> float:
-        if self.total_runs == 0:
-            return 0.0
-        return self.passed_runs / self.total_runs
-
-
-def load_scenario_results(
-    scenario_dir: Path,
-    *,
-    sweep_id: str | None = None,
-) -> dict[str, BackendResult]:
-    results: dict[str, BackendResult] = {}
-    for backend_dir in sorted(scenario_dir.iterdir()):
-        if not backend_dir.is_dir():
-            continue
-        timestamp_dirs = sorted(backend_dir.iterdir())
-        if not timestamp_dirs:
-            continue
-
-        target_dir: Path | None = None
-        if sweep_id:
-            for d in timestamp_dirs:
-                rg_path = d / "run-group.json"
-                if rg_path.exists():
-                    rg = json.loads(rg_path.read_text())
-                    if rg.get("sweep_id") == sweep_id:
-                        target_dir = d
-                        break
-        else:
-            target_dir = timestamp_dirs[-1]
-
-        if target_dir is None:
-            continue
-
-        result = _load_backend_result(backend_dir.name, target_dir)
-        if result is not None:
-            results[backend_dir.name] = result
-
-    return results
-
-
-def _load_backend_result(backend_name: str, timestamp_dir: Path) -> BackendResult | None:
-    rg_path = timestamp_dir / "run-group.json"
-
-    if rg_path.exists():
-        return _load_new_format(backend_name, timestamp_dir, rg_path)
-    elif (timestamp_dir / "verdict.json").exists():
-        return _load_old_format(backend_name, timestamp_dir)
-    return None
-
-
-def _load_new_format(backend_name: str, timestamp_dir: Path, rg_path: Path) -> BackendResult:
-    rg: dict[str, Any] = json.loads(rg_path.read_text())
-    run_dirs = sorted(
-        d for d in timestamp_dir.iterdir() if d.is_dir() and d.name.startswith("run-")
-    )
-
-    verdicts: list[Verdict] = []
-    metas: list[dict[str, Any]] = []
-    for run_dir in run_dirs:
-        verdict_path = run_dir / "verdict.json"
-        meta_path = run_dir / "meta.json"
-        if verdict_path.exists():
-            verdicts.append(Verdict.model_validate_json(verdict_path.read_text()))
-        if meta_path.exists():
-            metas.append(json.loads(meta_path.read_text()))
-
-    passed_runs = sum(1 for v in verdicts if v.passed)
-    errored_runs = sum(1 for r in rg.get("runs", []) if r.get("status") == "error")
-    avg_turns = sum(m.get("actor_turns", 0) for m in metas) / len(metas) if metas else 0.0
-
-    criterion_counts: dict[str, tuple[int, int]] = {}
-    for v in verdicts:
-        for c in v.criteria:
-            prev_passed, prev_total = criterion_counts.get(c.criterion, (0, 0))
-            criterion_counts[c.criterion] = (
-                prev_passed + (1 if c.verdict == "pass" else 0),
-                prev_total + 1,
-            )
-
-    return BackendResult(
-        backend=backend_name,
-        total_runs=len(verdicts),
-        passed_runs=passed_runs,
-        errored_runs=errored_runs,
-        avg_turns=round(avg_turns, 1),
-        criterion_counts=criterion_counts,
-        sweep_id=rg.get("sweep_id"),
-        timestamp=rg.get("timestamp"),
-        partial=rg.get("partial", False),
-    )
-
-
-def _load_old_format(backend_name: str, timestamp_dir: Path) -> BackendResult:
-    verdict = Verdict.model_validate_json((timestamp_dir / "verdict.json").read_text())
-    meta: dict[str, Any] = {}
-    meta_path = timestamp_dir / "meta.json"
-    if meta_path.exists():
-        meta = json.loads(meta_path.read_text())
-
-    criterion_counts: dict[str, tuple[int, int]] = {}
-    for c in verdict.criteria:
-        criterion_counts[c.criterion] = (1 if c.verdict == "pass" else 0, 1)
-
-    return BackendResult(
-        backend=backend_name,
-        total_runs=1,
-        passed_runs=1 if verdict.passed else 0,
-        errored_runs=0,
-        avg_turns=float(meta.get("actor_turns", 0)),
-        criterion_counts=criterion_counts,
-        sweep_id=None,
-        timestamp=None,
-        partial=False,
-    )
-
-
-def format_compare_output(
-    scenario: str,
-    results: dict[str, BackendResult],
-) -> str:
-    if not results:
-        return f"No results found for: {scenario}"
-
-    lines: list[str] = []
-    is_multi_run = any(r.total_runs > 1 for r in results.values())
-
-    if is_multi_run:
-        first = next(iter(results.values()))
-        lines.append(f"Scenario: {scenario}")
-        if first.sweep_id:
-            sweep_label = f"Sweep: {first.sweep_id}"
-            if first.timestamp:
-                date_str = first.timestamp.split("T")[0]
-                sweep_label += f" | {date_str}"
-            lines.append(sweep_label)
-        lines.append("")
-
-        header = f"{'':40s}"
-        sub_header = f"{'':40s}"
-        for name, r in results.items():
-            header += f" {name:>12s}"
-            sub_header += f" {'(n=' + str(r.total_runs) + ')':>12s}"
-        lines.append(header)
-        lines.append(sub_header)
-        lines.append("-" * len(header))
-
-        rate_line = f"{'Overall pass rate':40s}"
-        ci_line = f"{'  95% CI':40s}"
-        for r in results.values():
-            pct = f"{r.pass_rate * 100:.1f}%"
-            rate_line += f" {pct:>12s}"
-            lo, hi = wilson_ci(r.passed_runs, r.total_runs)
-            ci_str = f"[{lo * 100:.0f}, {hi * 100:.0f}]"
-            ci_line += f" {ci_str:>12s}"
-        lines.append(rate_line)
-        lines.append(ci_line)
-        lines.append("")
-
-        all_criteria: list[str] = []
-        seen: set[str] = set()
-        for r in results.values():
-            for crit in r.criterion_counts:
-                if crit not in seen:
-                    all_criteria.append(crit)
-                    seen.add(crit)
-
-        for crit in all_criteria:
-            crit_line = f"{crit[:40]:40s}"
-            for r in results.values():
-                passed, total = r.criterion_counts.get(crit, (0, 0))
-                crit_line += f" {str(passed) + '/' + str(total):>12s}"
-            lines.append(crit_line)
-
-        lines.append("")
-        avg_line = f"{'Avg turns':40s}"
-        err_line = f"{'Errors':40s}"
-        for r in results.values():
-            avg_line += f" {str(r.avg_turns):>12s}"
-            err_line += f" {str(r.errored_runs):>12s}"
-        lines.append(avg_line)
-        lines.append(err_line)
-
-        if any(r.total_runs < 10 for r in results.values()):
-            lines.append("")
-            lines.append("Note: CI is wide due to small sample size; consider --n 10+")
-
-        if any(r.partial for r in results.values()):
-            lines.append("")
-            lines.append("Warning: Sweep was interrupted — results are incomplete.")
-
-    else:
-        lines.append(f"Scenario: {scenario}")
-        lines.append("")
-        lines.append(f"{'Backend':20s} {'Result':8s} {'Score':7s} {'Turns':5s}")
-        lines.append("-" * 42)
-        for name, r in results.items():
-            result_str = "PASS" if r.passed_runs == r.total_runs else "FAIL"
-            total_criteria = sum(t for _, t in r.criterion_counts.values())
-            passed_criteria = sum(p for p, _ in r.criterion_counts.values())
-            score = f"{passed_criteria}/{total_criteria}"
-            turns_str = (
-                str(int(r.avg_turns)) if r.avg_turns == int(r.avg_turns) else str(r.avg_turns)
-            )
-            lines.append(f"{name:20s} {result_str:8s} {score:7s} {turns_str:5s}")
-
-        all_criteria = []
-        seen = set()
-        for r in results.values():
-            for crit in r.criterion_counts:
-                if crit not in seen:
-                    all_criteria.append(crit)
-                    seen.add(crit)
-
-        lines.append("")
-        header = f"{'':40s}"
-        for name in results:
-            header += f" {name:>12s}"
-        lines.append(header)
-        lines.append("-" * len(header))
-        for crit in all_criteria:
-            crit_line = f"{crit[:40]:40s}"
-            for r in results.values():
-                p, t = r.criterion_counts.get(crit, (0, 0))
-                icon = "PASS" if p == t and t > 0 else "FAIL"
-                crit_line += f" {icon:>12s}"
-            lines.append(crit_line)
-
-    return "\n".join(lines)

evals/drill/engine.py

@@ -1,377 +0,0 @@
-"""Engine: orchestrates the full Drill run lifecycle."""
-
-from __future__ import annotations
-
-import json
-import os
-import re
-import subprocess
-import time
-from dataclasses import dataclass, field
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-from drill.actor import Actor
-from drill.assertions import AssertionResult, run_verify_assertions
-from drill.backend import load_backend
-from drill.normalizer import (
-    NORMALIZERS,
-    collect_new_logs,
-    filter_codex_logs_by_cwd,
-    snapshot_log_dir,
-)
-from drill.session import TmuxSession
-from drill.setup import run_assertions, run_helpers
-from drill.verifier import Verifier
-
-
-@dataclass
-class VerifyConfig:
-    criteria: list[str] = field(default_factory=list)
-    assertions: list[str] = field(default_factory=list)
-    observe: bool = False
-
-
-@dataclass
-class ScenarioConfig:
-    scenario: str
-    description: str
-    user_posture: str
-    setup: dict[str, Any]
-    turns: list[dict[str, Any]]
-    limits: dict[str, Any]
-    verify: VerifyConfig
-
-    @classmethod
-    def from_yaml(cls, path: Path) -> ScenarioConfig:
-        with open(path) as f:
-            data = yaml.safe_load(f)
-        verify_data = data.get("verify", {})
-        return cls(
-            scenario=data["scenario"],
-            description=data.get("description", ""),
-            user_posture=data.get("user_posture", "naive"),
-            setup=data.get("setup", {}),
-            turns=data.get("turns", []),
-            limits=data.get("limits", {"max_turns": 20, "turn_timeout": 120}),
-            verify=VerifyConfig(
-                criteria=verify_data.get("criteria", []),
-                assertions=verify_data.get("assertions", []),
-                observe=verify_data.get("observe", False),
-            ),
-        )
-
-
-@dataclass
-class RunResult:
-    scenario: str
-    backend: str
-    timestamp: str
-    session_log: str
-    filesystem_json: str
-    tool_calls_jsonl: str
-    verdict_json: str
-    meta: dict[str, Any]
-
-    def save_artifacts(self, output_dir: Path) -> None:
-        output_dir.mkdir(parents=True, exist_ok=True)
-        (output_dir / "session.log").write_text(self.session_log)
-        (output_dir / "filesystem.json").write_text(self.filesystem_json)
-        (output_dir / "tool_calls.jsonl").write_text(self.tool_calls_jsonl)
-
-    def save_verdict(self, output_dir: Path) -> None:
-        output_dir.mkdir(parents=True, exist_ok=True)
-        (output_dir / "verdict.json").write_text(self.verdict_json)
-        (output_dir / "meta.json").write_text(json.dumps(self.meta, indent=2))
-
-    def save(self, output_dir: Path) -> None:
-        self.save_artifacts(output_dir)
-        self.save_verdict(output_dir)
-
-
-def snapshot_filesystem(workdir: Path) -> str:
-    files: list[str] = []
-    for f in sorted(workdir.rglob("*")):
-        if ".git" in f.parts:
-            continue
-        if f.is_file():
-            files.append(str(f.relative_to(workdir)))
-    git_status = _git_cmd(workdir, ["git", "status", "--short"])
-    branch = _git_cmd(workdir, ["git", "branch", "--show-current"])
-    worktree_list = _git_cmd(workdir, ["git", "worktree", "list"])
-    return json.dumps(
-        {
-            "files": files,
-            "git_status": git_status,
-            "branch": branch,
-            "worktree_list": worktree_list,
-        },
-        indent=2,
-    )
-
-
-class Engine:
-    def __init__(
-        self,
-        scenario_path: Path,
-        backend_name: str,
-        backends_dir: Path,
-        fixtures_dir: Path,
-        results_dir: Path,
-    ) -> None:
-        self.scenario = ScenarioConfig.from_yaml(scenario_path)
-        self.backend = load_backend(backend_name, backends_dir)
-        self.fixtures_dir = fixtures_dir
-        self.results_dir = results_dir
-
-    def run(self, *, output_dir: Path | None = None, run_suffix: str = "") -> RunResult:
-        start_time = time.time()
-        timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
-        self.backend.validate_env()
-        workdir = Path(f"/tmp/drill-{self.scenario.scenario}-{timestamp}{run_suffix}")
-        self._setup(workdir)
-        actual_workdir = workdir
-        override = self.scenario.setup.get("workdir_override")
-        if override:
-            resolved = override.replace("${WORKDIR_NAME}", workdir.name)
-            actual_workdir = (workdir / resolved).resolve()
-        # Run assertions in the actual workdir (after override)
-        assertions = self.scenario.setup.get("assertions", [])
-        if assertions:
-            run_assertions(assertions, actual_workdir)
-        session_name = f"drill-{self.scenario.scenario}-{timestamp}{run_suffix}"
-        session = TmuxSession(name=session_name, cols=self.backend.cols, rows=self.backend.rows)
-        log_dir = self._resolve_log_dir(actual_workdir)
-        log_snapshot = snapshot_log_dir(log_dir) if log_dir else set()
-        session_log, actor_turns = self._run_session(session, actual_workdir)
-        filesystem_json = snapshot_filesystem(actual_workdir)
-        tool_calls = self._collect_tool_calls(log_dir, log_snapshot, actual_workdir)
-        tool_calls_jsonl = "\n".join(json.dumps(tc) for tc in tool_calls)
-
-        # Write artifacts to disk before assertions (assertions read from disk)
-        if output_dir is None:
-            output_dir = self.results_dir / self.scenario.scenario / self.backend.name / timestamp
-        output_dir.mkdir(parents=True, exist_ok=True)
-        (output_dir / "session.log").write_text(session_log)
-        (output_dir / "filesystem.json").write_text(filesystem_json)
-        (output_dir / "tool_calls.jsonl").write_text(tool_calls_jsonl)
-
-        # Run deterministic assertions
-        assertion_results: list[AssertionResult] = []
-        if self.scenario.verify.assertions:
-            if not tool_calls_jsonl.strip():
-                assertion_results = [
-                    AssertionResult(
-                        command="<pre-check>",
-                        passed=False,
-                        exit_code=1,
-                        stdout="",
-                        stderr="tool_calls.jsonl is empty — session may have crashed",
-                    )
-                ]
-            else:
-                assertion_results = run_verify_assertions(
-                    self.scenario.verify.assertions,
-                    output_dir,
-                    actual_workdir,
-                )
-
-        # Run LLM verifier
-        verifier = Verifier()
-        verdict = verifier.verify(
-            session_log=session_log,
-            filesystem_json=filesystem_json,
-            tool_calls_jsonl=tool_calls_jsonl,
-            criteria=self.scenario.verify.criteria,
-        )
-
-        # Merge assertion results into verdict
-        for ar in assertion_results:
-            verdict.criteria.append(ar.to_criterion_result())
-
-        duration = time.time() - start_time
-        meta: dict[str, Any] = {
-            "scenario": self.scenario.scenario,
-            "backend": self.backend.name,
-            "backend_model": self.backend.model,
-            "user_posture": self.scenario.user_posture,
-            "timestamp": timestamp,
-            "duration_seconds": round(duration, 1),
-            "actor_turns": actor_turns,
-            "actor_model": "claude-sonnet-4-6",
-            "verifier_model": "claude-sonnet-4-6",
-        }
-        result = RunResult(
-            scenario=self.scenario.scenario,
-            backend=self.backend.name,
-            timestamp=timestamp,
-            session_log=session_log,
-            filesystem_json=filesystem_json,
-            tool_calls_jsonl=tool_calls_jsonl,
-            verdict_json=verdict.model_dump_json(indent=2),
-            meta=meta,
-        )
-        # Write verdict + meta (artifacts already on disk)
-        (output_dir / "verdict.json").write_text(result.verdict_json)
-        (output_dir / "meta.json").write_text(json.dumps(result.meta, indent=2))
-        return result
-
-    def _setup(self, workdir: Path) -> None:
-        # Scenario helpers first (create_base_repo needs to run before anything else)
-        helpers = self.scenario.setup.get("helpers", [])
-        run_helpers(helpers, workdir, self.fixtures_dir)
-        # Backend pre_run hooks after (e.g., codex symlink needs workdir to exist)
-        hooks_needing_superpowers_root = {"symlink_superpowers", "link_gemini_extension"}
-        for hook_name in self.backend.hooks.get("pre_run", []):
-            from setup_helpers import HELPER_REGISTRY
-
-            hook = HELPER_REGISTRY.get(hook_name)
-            if hook and hook_name in hooks_needing_superpowers_root:
-                hook(workdir, os.environ["SUPERPOWERS_ROOT"])  # ty: ignore[invalid-argument-type, too-many-positional-arguments, missing-argument]
-            elif hook:
-                hook(workdir)  # ty: ignore[invalid-argument-type, missing-argument]
-
-    def _run_session(self, session: TmuxSession, workdir: Path) -> tuple[str, int]:
-        session.create()
-        try:
-            cmd = self.backend.build_command(str(workdir))
-            session.launch(cmd, str(workdir))
-            self._wait_for_ready(session, timeout=self.backend.startup_timeout)
-            actor = Actor()
-            intents = [t["intent"] for t in self.scenario.turns]
-            actor.build_system_prompt(posture=self.scenario.user_posture, intents=intents)
-            max_turns = self.scenario.limits.get("max_turns", 20)
-            turn_timeout = self.backend.turn_timeout or self.scenario.limits.get(
-                "turn_timeout", 120
-            )
-            all_captures: list[str] = []
-            turn_count = 0
-            for turn in range(max_turns):
-                self._wait_for_ready(session, timeout=turn_timeout)
-                capture = session.capture()
-                all_captures.append(f"=== Turn {turn + 1} ===\n{capture}")
-                actor.append_capture(f"Terminal output:\n{capture}")
-                action = actor.decide()
-                turn_count += 1
-                if action.action == "done" or action.action == "stuck":
-                    break
-                elif action.action == "type":
-                    session.send_keys(action.text or "")
-                elif action.action == "key":
-                    session.send_special_key(action.key or "")
-            final_capture = session.capture()
-            all_captures.append(f"=== Final ===\n{final_capture}")
-            if self.backend.shutdown.startswith("<<KEY:"):
-                key = self.backend.shutdown[6:-2]
-                session.send_special_key(key)
-            else:
-                session.send_keys(self.backend.shutdown)
-            time.sleep(3)
-            return "\n".join(all_captures), turn_count
-        finally:
-            session.kill()
-
-    def _wait_for_ready(self, session: TmuxSession, timeout: float) -> None:
-        """Wait until the agent's terminal is ready for Actor input.
-
-        Returns when the terminal is quiescent AND matches the backend's
-        ready pattern. If the backend's busy pattern matches (spinner
-        visible, "Thinking...", timer counting), the deadline is extended
-        by small increments up to `max_busy_seconds` total. This prevents
-        the Actor from interrupting long-running subagent work (multi-file
-        implementation, parallel dispatch, etc.).
-
-        Exits silently if the final deadline (timeout + busy extensions)
-        passes without reaching a ready state.
-        """
-        quiescence = self.backend.quiescence_seconds
-        max_busy_extension = float(self.backend.max_busy_seconds)
-        start = time.time()
-        deadline = start + timeout
-        total_busy_extended = 0.0
-        last_output: str = ""
-        stable_since: float | None = None
-
-        while time.time() < deadline:
-            current = session.capture()
-            lines = current.strip().split("\n")
-            is_busy = any(self.backend.is_busy_line(line) for line in lines)
-
-            # If the agent is actively busy, extend the deadline so we
-            # don't time out mid-subagent-work. Extensions are capped at
-            # max_busy_seconds total across all extensions combined.
-            if is_busy:
-                remaining_budget = max_busy_extension - total_busy_extended
-                if remaining_budget > 0:
-                    # Ensure we have at least 30 more seconds of headroom.
-                    needed = 30.0 - (deadline - time.time())
-                    if needed > 0:
-                        grant = min(needed, remaining_budget)
-                        deadline += grant
-                        total_busy_extended += grant
-
-            # Strip animated elements so they don't reset the quiescence timer:
-            # - Time counters: "Thinking... (4m 1s)" or "(esc to cancel, 4m 1s)"
-            # - Braille spinner characters that rotate every frame
-            normalized = re.sub(r"\((?:esc to cancel, )?(?:\d+[hms]\s*)+\)", "(…)", current)
-            normalized = re.sub(r"[⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧⠶⠾⠽⠻⠿]", "·", normalized)
-            if normalized != last_output:
-                last_output = normalized
-                stable_since = time.time()
-            elif stable_since and (time.time() - stable_since) >= quiescence:
-                if is_busy:
-                    stable_since = None  # Reset — agent is still working
-                elif any(self.backend.is_ready_line(line) for line in lines):
-                    return
-            time.sleep(0.5)
-
-    def _resolve_log_dir(self, workdir: Path) -> Path | None:
-        """Resolve the log directory for the given backend and workdir.
-
-        Claude Code stores logs at ~/.claude/projects/<encoded-path>/
-        where the path is the real workdir with / replaced by -.
-        Codex stores logs at ~/.codex/sessions/.
-        """
-        if self.backend.family == "claude":
-            real_workdir = workdir.resolve()
-            encoded = str(real_workdir).replace("/", "-")
-            log_dir = Path.home() / ".claude" / "projects" / encoded
-            return log_dir
-        elif self.backend.family == "codex":
-            # Codex stores at ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl
-            return Path.home() / ".codex" / "sessions"
-        elif self.backend.family == "gemini":
-            # Gemini stores at ~/.gemini/tmp/<project-name>/chats/session-*.json
-            # Project name is the workdir basename, lowercased
-            project = workdir.resolve().name.lower()
-            return Path.home() / ".gemini" / "tmp" / project
-        pattern = self.backend.session_logs.get("pattern", "")
-        if not pattern:
-            return None
-        expanded = os.path.expanduser(pattern)
-        parts = expanded.split("*")[0].rstrip("/")
-        return Path(parts)
-
-    def _collect_tool_calls(
-        self, log_dir: Path | None, snapshot: set[str], workdir: Path
-    ) -> list[dict[str, Any]]:
-        if log_dir is None:
-            return []
-        new_files = collect_new_logs(log_dir, snapshot)
-        if self.backend.family == "codex":
-            new_files = filter_codex_logs_by_cwd(new_files, str(workdir.resolve()))
-        normalizer = NORMALIZERS.get(self.backend.family)
-        if not normalizer:
-            return []
-        results: list[dict[str, Any]] = []
-        for log_file in new_files:
-            results.extend(normalizer(log_file.read_text()))
-        return results
-
-
-def _git_cmd(workdir: Path, cmd: list[str]) -> str:
-    result = subprocess.run(cmd, cwd=workdir, capture_output=True, text=True)
-    return result.stdout.strip()

evals/drill/normalizer.py

@@ -1,228 +0,0 @@
-"""Normalizes backend-specific session logs to a common tool call schema."""
-
-from __future__ import annotations
-
-import json
-from collections.abc import Callable
-from pathlib import Path
-from typing import Any
-
-NATIVE_TOOLS: set[str] = {
-    "EnterWorktree",
-    "ExitWorktree",
-    "EnterPlanMode",
-    "ExitPlanMode",
-    "TaskCreate",
-    "TaskUpdate",
-    "TaskList",
-    "TaskGet",
-    "Skill",
-    "Agent",
-    "Read",
-    "Write",
-    "Edit",
-    "Glob",
-    "Grep",
-}
-
-LOG_EXTENSIONS: tuple[str, ...] = ("*.jsonl", "*.json")
-
-
-def snapshot_log_dir(log_dir: Path) -> set[str]:
-    """Snapshot all session log files in a log directory (recursive)."""
-    if not log_dir.exists():
-        return set()
-    files: set[str] = set()
-    for ext in LOG_EXTENSIONS:
-        files.update(str(f.relative_to(log_dir)) for f in log_dir.rglob(ext))
-    return files
-
-
-def collect_new_logs(log_dir: Path, snapshot: set[str]) -> list[Path]:
-    """Find session log files created after the snapshot (recursive)."""
-    if not log_dir.exists():
-        return []
-    current: dict[str, Path] = {}
-    for ext in LOG_EXTENSIONS:
-        current.update({str(f.relative_to(log_dir)): f for f in log_dir.rglob(ext)})
-    new_keys: set[str] = set(current.keys()) - snapshot
-    return [current[k] for k in sorted(new_keys)]
-
-
-def filter_codex_logs_by_cwd(paths: list[Path], target_cwd: str) -> list[Path]:
-    """Drop codex rollouts whose session_meta.cwd doesn't match target_cwd.
-
-    Codex stores all sessions under a shared ~/.codex/sessions/ tree, so when
-    multiple drill scenarios run in parallel each one's snapshot diff sees every
-    other run's rollouts. Each rollout's first line is a `session_meta` event
-    that records the cwd the codex CLI was launched in — use it to attribute
-    rollouts to the run that produced them.
-    """
-    matched: list[Path] = []
-    for path in paths:
-        try:
-            with path.open() as f:
-                first_line = f.readline()
-            entry = json.loads(first_line)
-        except (OSError, json.JSONDecodeError):
-            continue
-        if entry.get("type") != "session_meta":
-            continue
-        cwd = entry.get("payload", {}).get("cwd", "")
-        if cwd == target_cwd:
-            matched.append(path)
-    return matched
-
-
-def normalize_claude_logs(raw_content: str) -> list[dict[str, Any]]:
-    """Normalize Claude Code session logs.
-
-    CC logs are JSONL where assistant messages have:
-    {"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "...",
-    "input": {...}}]}}
-    """
-    results: list[dict[str, Any]] = []
-    for line in raw_content.strip().split("\n"):
-        if not line.strip():
-            continue
-        try:
-            entry = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-        # Handle nested CC format: assistant messages contain tool_use in content array
-        if entry.get("type") == "assistant":
-            message = entry.get("message", {})
-            for block in message.get("content", []):
-                if block.get("type") == "tool_use":
-                    tool_name = block.get("name", "")
-                    source = "native" if tool_name in NATIVE_TOOLS else "shell"
-                    results.append(
-                        {"tool": tool_name, "args": block.get("input", {}), "source": source}
-                    )
-        # Also handle flat format (for test compatibility)
-        elif entry.get("type") == "tool_use":
-            tool_name = entry.get("name", "")
-            source = "native" if tool_name in NATIVE_TOOLS else "shell"
-            results.append({"tool": tool_name, "args": entry.get("input", {}), "source": source})
-    return results
-
-
-def normalize_codex_logs(raw_content: str) -> list[dict[str, Any]]:
-    """Normalize Codex rollout logs.
-
-    Codex logs use: {"type": "response_item", "payload": {"type": "function_call", ...}}
-    Tool calls are "function_call" with name "exec_command" (shell) or other names.
-    """
-    results: list[dict[str, Any]] = []
-    for line in raw_content.strip().split("\n"):
-        if not line.strip():
-            continue
-        try:
-            entry = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-        if entry.get("type") != "response_item":
-            continue
-        # Codex uses "payload" not "item"
-        payload = entry.get("payload", entry.get("item", {}))
-        payload_type = payload.get("type", "")
-        if payload_type == "function_call":
-            name = payload.get("name", "")
-            raw_args = payload.get("arguments", "{}")
-            # Arguments are JSON-encoded strings in codex
-            if isinstance(raw_args, str):
-                try:
-                    args = json.loads(raw_args)
-                except json.JSONDecodeError:
-                    args = {"raw": raw_args}
-            else:
-                args = raw_args
-            # exec_command is codex's shell tool
-            if name == "exec_command":
-                results.append(
-                    {"tool": "Bash", "args": {"command": args.get("cmd", "")}, "source": "shell"}
-                )
-            elif name == "apply_patch":
-                results.append({"tool": "Edit", "args": args, "source": "native"})
-            else:
-                source = "native" if name in NATIVE_TOOLS else "shell"
-                results.append({"tool": name, "args": args, "source": source})
-        elif payload_type == "local_shell_call":
-            action = payload.get("action", {})
-            cmd = action.get("command", [])
-            cmd_str = " ".join(cmd) if isinstance(cmd, list) else str(cmd)
-            results.append({"tool": "Bash", "args": {"command": cmd_str}, "source": "shell"})
-    return results
-
-
-# Reverse mapping: Gemini tool names → Claude Code canonical names
-GEMINI_TOOL_MAP: dict[str, str] = {
-    "run_shell_command": "Bash",
-    "read_file": "Read",
-    "write_file": "Write",
-    "replace": "Edit",
-    "grep_search": "Grep",
-    "glob": "Glob",
-    "activate_skill": "Skill",
-    "google_web_search": "WebSearch",
-    "web_fetch": "WebFetch",
-    "write_todos": "TodoWrite",
-    "list_directory": "Glob",
-    "enter_plan_mode": "EnterPlanMode",
-    "exit_plan_mode": "ExitPlanMode",
-}
-
-
-def normalize_gemini_logs(raw_content: str) -> list[dict[str, Any]]:
-    """Normalize Gemini CLI session logs.
-
-    Gemini logs may be a single JSON file with a messages array, or JSONL
-    session files in newer CLI versions. Each "gemini" message may have a
-    toolCalls array:
-    {"name": "run_shell_command", "args": {"command": "..."}, "status": "success"}
-    """
-    results: list[dict[str, Any]] = []
-    messages: list[dict[str, Any]] = []
-    try:
-        data = json.loads(raw_content)
-    except json.JSONDecodeError:
-        for line in raw_content.strip().split("\n"):
-            if not line.strip():
-                continue
-            try:
-                entry = json.loads(line)
-            except json.JSONDecodeError:
-                continue
-            if isinstance(entry, dict):
-                messages.append(entry)
-    else:
-        if isinstance(data, dict) and "messages" in data:
-            messages = [m for m in data.get("messages", []) if isinstance(m, dict)]
-        elif isinstance(data, dict):
-            messages = [data]
-        elif isinstance(data, list):
-            messages = [m for m in data if isinstance(m, dict)]
-
-    seen_tool_calls: set[str] = set()
-    for message in messages:
-        if message.get("type") != "gemini":
-            continue
-        for tc in message.get("toolCalls", []):
-            tool_call_id = tc.get("id")
-            if tool_call_id and tool_call_id in seen_tool_calls:
-                continue
-            if tool_call_id:
-                seen_tool_calls.add(tool_call_id)
-            gemini_name = tc.get("name", "")
-            canonical = GEMINI_TOOL_MAP.get(gemini_name, gemini_name)
-            args = tc.get("args", {})
-            source = "native" if canonical in NATIVE_TOOLS else "shell"
-            results.append({"tool": canonical, "args": args, "source": source})
-    return results
-
-
-NORMALIZERS: dict[str, Callable[[str], list[dict[str, Any]]]] = {
-    "claude": normalize_claude_logs,
-    "codex": normalize_codex_logs,
-    "gemini": normalize_gemini_logs,
-}

evals/drill/session.py

@@ -1,88 +0,0 @@
-"""tmux session management for driving agent CLI sessions."""
-
-from __future__ import annotations
-
-import subprocess
-import time
-
-
-class TmuxSession:
-    def __init__(self, name: str, cols: int = 200, rows: int = 50) -> None:
-        self.name = name
-        self.cols = cols
-        self.rows = rows
-
-    def create(self) -> None:
-        subprocess.run(
-            [
-                "tmux",
-                "new-session",
-                "-d",
-                "-s",
-                self.name,
-                "-x",
-                str(self.cols),
-                "-y",
-                str(self.rows),
-            ],
-            check=True,
-        )
-
-    def launch(self, command: list[str], cwd: str) -> None:
-        cmd_str = " ".join(command)
-        self.send_keys(f"cd {cwd} && {cmd_str}")
-
-    def send_keys(self, text: str) -> None:
-        if text:
-            buffer_name = f"{self.name}-input"
-            subprocess.run(
-                ["tmux", "set-buffer", "-b", buffer_name, text],
-                check=True,
-            )
-            subprocess.run(
-                ["tmux", "paste-buffer", "-d", "-b", buffer_name, "-t", self.name],
-                check=True,
-            )
-            time.sleep(0.1)
-
-        subprocess.run(
-            ["tmux", "send-keys", "-t", self.name, "Enter"],
-            check=True,
-        )
-
-    def send_special_key(self, key: str) -> None:
-        key_map = {
-            "ctrl-c": "C-c",
-            "ctrl-d": "C-d",
-            "ctrl-z": "C-z",
-            "enter": "Enter",
-            "escape": "Escape",
-        }
-        tmux_key = key_map.get(key, key)
-        subprocess.run(
-            ["tmux", "send-keys", "-t", self.name, tmux_key],
-            check=True,
-        )
-
-    def capture(self) -> str:
-        result = subprocess.run(
-            ["tmux", "capture-pane", "-t", self.name, "-p"],
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        return result.stdout
-
-    def is_process_alive(self) -> bool:
-        result = subprocess.run(
-            ["tmux", "list-panes", "-t", self.name, "-F", "#{pane_dead}"],
-            capture_output=True,
-            text=True,
-        )
-        return result.stdout.strip() == "0"
-
-    def kill(self) -> None:
-        subprocess.run(
-            ["tmux", "kill-session", "-t", self.name],
-            capture_output=True,
-        )

evals/drill/setup.py

@@ -1,43 +0,0 @@
-from __future__ import annotations
-
-import subprocess
-from pathlib import Path
-
-from setup_helpers import HELPER_REGISTRY
-from setup_helpers.base import create_base_repo
-
-
-def clone_template(template_dir: Path, workdir: Path) -> None:
-    """Clone (or build) template_dir into workdir with full git history."""
-    create_base_repo(workdir, template_dir)
-
-
-def run_helpers(helper_names: list[str], workdir: Path, fixtures_dir: Path) -> None:
-    for name in helper_names:
-        helper = HELPER_REGISTRY.get(name)
-        if helper is None:
-            raise ValueError(f"Unknown setup helper: {name}")
-        if name == "create_base_repo":
-            helper(workdir, fixtures_dir / "template-repo")  # ty: ignore[invalid-argument-type, too-many-positional-arguments, missing-argument]
-        elif name == "symlink_superpowers":
-            import os
-
-            helper(workdir, os.environ["SUPERPOWERS_ROOT"])  # ty: ignore[invalid-argument-type, too-many-positional-arguments, missing-argument]
-        else:
-            helper(workdir)  # ty: ignore[invalid-argument-type, missing-argument]
-
-
-def run_assertions(assertions: list[str], workdir: Path) -> None:
-    for assertion in assertions:
-        result = subprocess.run(
-            assertion,
-            shell=True,
-            cwd=workdir,
-            capture_output=True,
-            text=True,
-        )
-        if result.returncode != 0:
-            raise AssertionError(
-                f"Setup assertion failed: {assertion}\n"
-                f"stdout: {result.stdout}\nstderr: {result.stderr}"
-            )

evals/drill/stats.py

@@ -1,17 +0,0 @@
-"""Statistical utilities for drill result analysis."""
-
-from __future__ import annotations
-
-import math
-
-
-def wilson_ci(passed: int, total: int, z: float = 1.96) -> tuple[float, float]:
-    if total == 0:
-        return (0.0, 0.0)
-    if passed > total:
-        passed = total
-    p = passed / total
-    denom = 1 + z**2 / total
-    center = (p + z**2 / (2 * total)) / denom
-    margin = (z / denom) * math.sqrt(p * (1 - p) / total + z**2 / (4 * total**2))
-    return (max(0.0, center - margin), min(1.0, center + margin))

evals/drill/sweep.py

@@ -1,159 +0,0 @@
-"""Sweep orchestrator: runs scenarios N times across multiple backends."""
-
-from __future__ import annotations
-
-import glob as glob_mod
-import json
-import shutil
-import time
-from dataclasses import asdict, dataclass, field
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-from drill.engine import Engine, RunResult
-from drill.verifier import Verdict
-
-
-@dataclass
-class RunStatus:
-    index: int
-    status: str  # "pass", "fail", "error"
-    duration: float
-    error: str | None = None
-
-
-@dataclass
-class RunGroup:
-    scenario: str
-    backend: str
-    n: int
-    timestamp: str
-    sweep_id: str
-    runs: list[RunStatus] = field(default_factory=list)
-    partial: bool = False
-
-
-def write_run_group(group: RunGroup, output_dir: Path) -> None:
-    output_dir.mkdir(parents=True, exist_ok=True)
-    data: dict[str, Any] = {
-        "scenario": group.scenario,
-        "backend": group.backend,
-        "n": group.n,
-        "timestamp": group.timestamp,
-        "sweep_id": group.sweep_id,
-        "partial": group.partial,
-        "runs": [
-            {k: v for k, v in asdict(r).items() if k != "error" or v is not None}
-            for r in group.runs
-        ],
-    }
-    (output_dir / "run-group.json").write_text(json.dumps(data, indent=2))
-
-
-class Sweep:
-    def __init__(
-        self,
-        scenario_path: Path,
-        backend_names: list[str],
-        backends_dir: Path,
-        fixtures_dir: Path,
-        results_dir: Path,
-        n: int,
-        sweep_id: str,
-    ) -> None:
-        self.scenario_path = scenario_path
-        self.backend_names = backend_names
-        self.backends_dir = backends_dir
-        self.fixtures_dir = fixtures_dir
-        self.results_dir = results_dir
-        self.n = n
-        self.sweep_id = sweep_id
-        self._scenario_name_cache: str | None = None
-
-    def validate_backends(self) -> None:
-        for name in self.backend_names:
-            path = self.backends_dir / f"{name}.yaml"
-            if not path.exists():
-                raise FileNotFoundError(f"Backend config not found: {path}")
-
-    def run_all(self) -> list[RunGroup]:
-        self.validate_backends()
-        groups: list[RunGroup] = []
-        for backend_name in self.backend_names:
-            group = self._run_backend(backend_name)
-            groups.append(group)
-        return groups
-
-    def _run_backend(self, backend_name: str) -> RunGroup:
-        timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
-        group_dir = (
-            self.results_dir / self.scenario_name / backend_name / f"{timestamp}-{self.sweep_id}"
-        )
-        group_dir.mkdir(parents=True, exist_ok=True)
-
-        group = RunGroup(
-            scenario=self.scenario_name,
-            backend=backend_name,
-            n=self.n,
-            timestamp=timestamp,
-            sweep_id=self.sweep_id,
-        )
-
-        try:
-            for i in range(self.n):
-                run_status = self._run_single(backend_name, group_dir, i, timestamp)
-                group.runs.append(run_status)
-        except KeyboardInterrupt:
-            group.partial = True
-        finally:
-            write_run_group(group, group_dir)
-
-        return group
-
-    def _run_single(
-        self, backend_name: str, group_dir: Path, index: int, timestamp: str
-    ) -> RunStatus:
-        run_suffix = f"-run-{index:02d}"
-        run_dir = group_dir / f"run-{index:02d}"
-        start = time.time()
-
-        try:
-            engine = Engine(
-                scenario_path=self.scenario_path,
-                backend_name=backend_name,
-                backends_dir=self.backends_dir,
-                fixtures_dir=self.fixtures_dir,
-                results_dir=self.results_dir,
-            )
-            result: RunResult = engine.run(output_dir=run_dir, run_suffix=run_suffix)
-            verdict = Verdict.model_validate_json(result.verdict_json)
-            duration = time.time() - start
-            status = "pass" if verdict.passed else "fail"
-            return RunStatus(index=index, status=status, duration=round(duration, 1))
-        except KeyboardInterrupt:
-            raise
-        except Exception as e:
-            duration = time.time() - start
-            return RunStatus(
-                index=index,
-                status="error",
-                duration=round(duration, 1),
-                error=str(e),
-            )
-        finally:
-            pattern = f"/tmp/drill-*-{timestamp}{run_suffix}"
-            for d in glob_mod.glob(pattern):
-                p = Path(d)
-                if p.is_dir():
-                    shutil.rmtree(p, ignore_errors=True)
-
-    @property
-    def scenario_name(self) -> str:
-        if self._scenario_name_cache is None:
-            with open(self.scenario_path) as f:
-                data = yaml.safe_load(f)
-            self._scenario_name_cache = data["scenario"]
-        return self._scenario_name_cache

evals/drill/verifier.py

@@ -1,93 +0,0 @@
-"""Verifier LLM: evaluates agent session against criteria."""
-
-from __future__ import annotations
-
-from pathlib import Path
-
-import anthropic
-from pydantic import BaseModel
-
-
-class CriterionResult(BaseModel):
-    criterion: str
-    verdict: str
-    evidence: str
-    rationale: str
-    source: str = "judge"
-
-
-class Verdict(BaseModel):
-    criteria: list[CriterionResult]
-    observations: list[str]
-    summary: str
-
-    @property
-    def score(self) -> str:
-        passed = sum(1 for c in self.criteria if c.verdict == "pass")
-        return f"{passed}/{len(self.criteria)}"
-
-    @property
-    def passed(self) -> bool:
-        return all(c.verdict == "pass" for c in self.criteria)
-
-
-class Verifier:
-    MAX_RETRIES = 3
-
-    def __init__(self, model: str = "claude-sonnet-4-6", temperature: float = 0.0) -> None:
-        self.model = model
-        self.temperature = temperature
-        self._client: anthropic.Anthropic = anthropic.Anthropic()
-
-    def build_system_prompt(self) -> str:
-        template_path = Path(__file__).parent.parent / "prompts" / "verifier.md"
-        return template_path.read_text()
-
-    def verify(
-        self,
-        session_log: str,
-        filesystem_json: str,
-        tool_calls_jsonl: str,
-        criteria: list[str],
-    ) -> Verdict:
-        system = self.build_system_prompt()
-        user_content = (
-            "## Terminal Session Log\n\n"
-            f"```\n{session_log}\n```\n\n"
-            "## Filesystem State\n\n"
-            f"```json\n{filesystem_json}\n```\n\n"
-            "## Tool Call Log\n\n"
-            f"```jsonl\n{tool_calls_jsonl}\n```\n\n"
-            "## Criteria to Evaluate\n\n" + "\n".join(f"- {c}" for c in criteria)
-        )
-        for attempt in range(self.MAX_RETRIES):
-            response = self._client.messages.create(
-                model=self.model,
-                max_tokens=4096,
-                temperature=self.temperature,
-                system=system,
-                messages=[{"role": "user", "content": user_content}],
-            )
-            text = response.content[0].text  # ty: ignore[unresolved-attribute]
-            json_str = _extract_json(text)
-            try:
-                return Verdict.model_validate_json(json_str)
-            except Exception:
-                if attempt == self.MAX_RETRIES - 1:
-                    raise
-                continue
-        raise RuntimeError("Verifier failed to return valid JSON")
-
-
-def _extract_json(text: str) -> str:
-    if "```json" in text:
-        start = text.index("```json") + 7
-        end = text.index("```", start)
-        return text[start:end].strip()
-    if "```" in text:
-        start = text.index("```") + 3
-        end = text.index("```", start)
-        return text[start:end].strip()
-    start = text.index("{")
-    end = text.rindex("}") + 1
-    return text[start:end]

evals/fixtures/sdd-go-fractals/design.md

@@ -1,81 +0,0 @@
-# Go Fractals CLI - Design
-
-## Overview
-
-A command-line tool that generates ASCII art fractals. Supports two fractal types with configurable output.
-
-## Usage
-
-```bash
-# Sierpinski triangle
-fractals sierpinski --size 32 --depth 5
-
-# Mandelbrot set
-fractals mandelbrot --width 80 --height 24 --iterations 100
-
-# Custom character
-fractals sierpinski --size 16 --char '#'
-
-# Help
-fractals --help
-fractals sierpinski --help
-```
-
-## Commands
-
-### `sierpinski`
-
-Generates a Sierpinski triangle using recursive subdivision.
-
-Flags:
-- `--size` (default: 32) - Width of the triangle base in characters
-- `--depth` (default: 5) - Recursion depth
-- `--char` (default: '*') - Character to use for filled points
-
-Output: Triangle printed to stdout, one line per row.
-
-### `mandelbrot`
-
-Renders the Mandelbrot set as ASCII art. Maps iteration count to characters.
-
-Flags:
-- `--width` (default: 80) - Output width in characters
-- `--height` (default: 24) - Output height in characters
-- `--iterations` (default: 100) - Maximum iterations for escape calculation
-- `--char` (default: gradient) - Single character, or omit for gradient " .:-=+*#%@"
-
-Output: Rectangle printed to stdout.
-
-## Architecture
-
-```
-cmd/
-  fractals/
-    main.go           # Entry point, CLI setup
-internal/
-  sierpinski/
-    sierpinski.go     # Algorithm
-    sierpinski_test.go
-  mandelbrot/
-    mandelbrot.go     # Algorithm
-    mandelbrot_test.go
-  cli/
-    root.go           # Root command, help
-    sierpinski.go     # Sierpinski subcommand
-    mandelbrot.go     # Mandelbrot subcommand
-```
-
-## Dependencies
-
-- Go 1.21+
-- `github.com/spf13/cobra` for CLI
-
-## Acceptance Criteria
-
-1. `fractals --help` shows usage
-2. `fractals sierpinski` outputs a recognizable triangle
-3. `fractals mandelbrot` outputs a recognizable Mandelbrot set
-4. `--size`, `--width`, `--height`, `--depth`, `--iterations` flags work
-5. `--char` customizes output character
-6. Invalid inputs produce clear error messages
-7. All tests pass

evals/fixtures/sdd-go-fractals/plan.md

@@ -1,172 +0,0 @@
-# Go Fractals CLI - Implementation Plan
-
-Execute this plan using the `superpowers:subagent-driven-development` skill.
-
-## Context
-
-Building a CLI tool that generates ASCII fractals. See `design.md` for full specification.
-
-## Tasks
-
-### Task 1: Project Setup
-
-Create the Go module and directory structure.
-
-**Do:**
-- Initialize `go.mod` with module name `github.com/superpowers-test/fractals`
-- Create directory structure: `cmd/fractals/`, `internal/sierpinski/`, `internal/mandelbrot/`, `internal/cli/`
-- Create minimal `cmd/fractals/main.go` that prints "fractals cli"
-- Add `github.com/spf13/cobra` dependency
-
-**Verify:**
-- `go build ./cmd/fractals` succeeds
-- `./fractals` prints "fractals cli"
-
----
-
-### Task 2: CLI Framework with Help
-
-Set up Cobra root command with help output.
-
-**Do:**
-- Create `internal/cli/root.go` with root command
-- Configure help text showing available subcommands
-- Wire root command into `main.go`
-
-**Verify:**
-- `./fractals --help` shows usage with "sierpinski" and "mandelbrot" listed as available commands
-- `./fractals` (no args) shows help
-
----
-
-### Task 3: Sierpinski Algorithm
-
-Implement the Sierpinski triangle generation algorithm.
-
-**Do:**
-- Create `internal/sierpinski/sierpinski.go`
-- Implement `Generate(size, depth int, char rune) []string` that returns lines of the triangle
-- Use recursive midpoint subdivision algorithm
-- Create `internal/sierpinski/sierpinski_test.go` with tests:
-  - Small triangle (size=4, depth=2) matches expected output
-  - Size=1 returns single character
-  - Depth=0 returns filled triangle
-
-**Verify:**
-- `go test ./internal/sierpinski/...` passes
-
----
-
-### Task 4: Sierpinski CLI Integration
-
-Wire the Sierpinski algorithm to a CLI subcommand.
-
-**Do:**
-- Create `internal/cli/sierpinski.go` with `sierpinski` subcommand
-- Add flags: `--size` (default 32), `--depth` (default 5), `--char` (default '*')
-- Call `sierpinski.Generate()` and print result to stdout
-
-**Verify:**
-- `./fractals sierpinski` outputs a triangle
-- `./fractals sierpinski --size 16 --depth 3` outputs smaller triangle
-- `./fractals sierpinski --help` shows flag documentation
-
----
-
-### Task 5: Mandelbrot Algorithm
-
-Implement the Mandelbrot set ASCII renderer.
-
-**Do:**
-- Create `internal/mandelbrot/mandelbrot.go`
-- Implement `Render(width, height, maxIter int, char string) []string`
-- Map complex plane region (-2.5 to 1.0 real, -1.0 to 1.0 imaginary) to output dimensions
-- Map iteration count to character gradient " .:-=+*#%@" (or single char if provided)
-- Create `internal/mandelbrot/mandelbrot_test.go` with tests:
-  - Output dimensions match requested width/height
-  - Known point inside set (0,0) maps to max-iteration character
-  - Known point outside set (2,0) maps to low-iteration character
-
-**Verify:**
-- `go test ./internal/mandelbrot/...` passes
-
----
-
-### Task 6: Mandelbrot CLI Integration
-
-Wire the Mandelbrot algorithm to a CLI subcommand.
-
-**Do:**
-- Create `internal/cli/mandelbrot.go` with `mandelbrot` subcommand
-- Add flags: `--width` (default 80), `--height` (default 24), `--iterations` (default 100), `--char` (default "")
-- Call `mandelbrot.Render()` and print result to stdout
-
-**Verify:**
-- `./fractals mandelbrot` outputs recognizable Mandelbrot set
-- `./fractals mandelbrot --width 40 --height 12` outputs smaller version
-- `./fractals mandelbrot --help` shows flag documentation
-
----
-
-### Task 7: Character Set Configuration
-
-Ensure `--char` flag works consistently across both commands.
-
-**Do:**
-- Verify Sierpinski `--char` flag passes character to algorithm
-- For Mandelbrot, `--char` should use single character instead of gradient
-- Add tests for custom character output
-
-**Verify:**
-- `./fractals sierpinski --char '#'` uses '#' character
-- `./fractals mandelbrot --char '.'` uses '.' for all filled points
-- Tests pass
-
----
-
-### Task 8: Input Validation and Error Handling
-
-Add validation for invalid inputs.
-
-**Do:**
-- Sierpinski: size must be > 0, depth must be >= 0
-- Mandelbrot: width/height must be > 0, iterations must be > 0
-- Return clear error messages for invalid inputs
-- Add tests for error cases
-
-**Verify:**
-- `./fractals sierpinski --size 0` prints error, exits non-zero
-- `./fractals mandelbrot --width -1` prints error, exits non-zero
-- Error messages are clear and helpful
-
----
-
-### Task 9: Integration Tests
-
-Add integration tests that invoke the CLI.
-
-**Do:**
-- Create `cmd/fractals/main_test.go` or `test/integration_test.go`
-- Test full CLI invocation for both commands
-- Verify output format and exit codes
-- Test error cases return non-zero exit
-
-**Verify:**
-- `go test ./...` passes all tests including integration tests
-
----
-
-### Task 10: README
-
-Document usage and examples.
-
-**Do:**
-- Create `README.md` with:
-  - Project description
-  - Installation: `go install ./cmd/fractals`
-  - Usage examples for both commands
-  - Example output (small samples)
-
-**Verify:**
-- README accurately describes the tool
-- Examples in README actually work

evals/fixtures/sdd-svelte-todo/design.md

@@ -1,70 +0,0 @@
-# Svelte Todo List - Design
-
-## Overview
-
-A simple todo list application built with Svelte. Supports creating, completing, and deleting todos with localStorage persistence.
-
-## Features
-
-- Add new todos
-- Mark todos as complete/incomplete
-- Delete todos
-- Filter by: All / Active / Completed
-- Clear all completed todos
-- Persist to localStorage
-- Show count of remaining items
-
-## User Interface
-
-```
-┌─────────────────────────────────────────┐
-│  Svelte Todos                           │
-├─────────────────────────────────────────┤
-│  [________________________] [Add]       │
-├─────────────────────────────────────────┤
-│  [ ] Buy groceries                  [x] │
-│  [✓] Walk the dog                   [x] │
-│  [ ] Write code                     [x] │
-├─────────────────────────────────────────┤
-│  2 items left                           │
-│  [All] [Active] [Completed]  [Clear ✓]  │
-└─────────────────────────────────────────┘
-```
-
-## Components
-
-```
-src/
-  App.svelte           # Main app, state management
-  lib/
-    TodoInput.svelte   # Text input + Add button
-    TodoList.svelte    # List container
-    TodoItem.svelte    # Single todo with checkbox, text, delete
-    FilterBar.svelte   # Filter buttons + clear completed
-    store.ts           # Svelte store for todos
-    storage.ts         # localStorage persistence
-```
-
-## Data Model
-
-```typescript
-interface Todo {
-  id: string;        // UUID
-  text: string;      // Todo text
-  completed: boolean;
-}
-
-type Filter = 'all' | 'active' | 'completed';
-```
-
-## Acceptance Criteria
-
-1. Can add a todo by typing and pressing Enter or clicking Add
-2. Can toggle todo completion by clicking checkbox
-3. Can delete a todo by clicking X button
-4. Filter buttons show correct subset of todos
-5. "X items left" shows count of incomplete todos
-6. "Clear completed" removes all completed todos
-7. Todos persist across page refresh (localStorage)
-8. Empty state shows helpful message
-9. All tests pass

evals/fixtures/sdd-svelte-todo/plan.md

@@ -1,222 +0,0 @@
-# Svelte Todo List - Implementation Plan
-
-Execute this plan using the `superpowers:subagent-driven-development` skill.
-
-## Context
-
-Building a todo list app with Svelte. See `design.md` for full specification.
-
-## Tasks
-
-### Task 1: Project Setup
-
-Create the Svelte project with Vite.
-
-**Do:**
-- Run `npm create vite@latest . -- --template svelte-ts`
-- Install dependencies with `npm install`
-- Verify dev server works
-- Clean up default Vite template content from App.svelte
-
-**Verify:**
-- `npm run dev` starts server
-- App shows minimal "Svelte Todos" heading
-- `npm run build` succeeds
-
----
-
-### Task 2: Todo Store
-
-Create the Svelte store for todo state management.
-
-**Do:**
-- Create `src/lib/store.ts`
-- Define `Todo` interface with id, text, completed
-- Create writable store with initial empty array
-- Export functions: `addTodo(text)`, `toggleTodo(id)`, `deleteTodo(id)`, `clearCompleted()`
-- Create `src/lib/store.test.ts` with tests for each function
-
-**Verify:**
-- Tests pass: `npm run test` (install vitest if needed)
-
----
-
-### Task 3: localStorage Persistence
-
-Add persistence layer for todos.
-
-**Do:**
-- Create `src/lib/storage.ts`
-- Implement `loadTodos(): Todo[]` and `saveTodos(todos: Todo[])`
-- Handle JSON parse errors gracefully (return empty array)
-- Integrate with store: load on init, save on change
-- Add tests for load/save/error handling
-
-**Verify:**
-- Tests pass
-- Manual test: add todo, refresh page, todo persists
-
----
-
-### Task 4: TodoInput Component
-
-Create the input component for adding todos.
-
-**Do:**
-- Create `src/lib/TodoInput.svelte`
-- Text input bound to local state
-- Add button calls `addTodo()` and clears input
-- Enter key also submits
-- Disable Add button when input is empty
-- Add component tests
-
-**Verify:**
-- Tests pass
-- Component renders input and button
-
----
-
-### Task 5: TodoItem Component
-
-Create the single todo item component.
-
-**Do:**
-- Create `src/lib/TodoItem.svelte`
-- Props: `todo: Todo`
-- Checkbox toggles completion (calls `toggleTodo`)
-- Text with strikethrough when completed
-- Delete button (X) calls `deleteTodo`
-- Add component tests
-
-**Verify:**
-- Tests pass
-- Component renders checkbox, text, delete button
-
----
-
-### Task 6: TodoList Component
-
-Create the list container component.
-
-**Do:**
-- Create `src/lib/TodoList.svelte`
-- Props: `todos: Todo[]`
-- Renders TodoItem for each todo
-- Shows "No todos yet" when empty
-- Add component tests
-
-**Verify:**
-- Tests pass
-- Component renders list of TodoItems
-
----
-
-### Task 7: FilterBar Component
-
-Create the filter and status bar component.
-
-**Do:**
-- Create `src/lib/FilterBar.svelte`
-- Props: `todos: Todo[]`, `filter: Filter`, `onFilterChange: (f: Filter) => void`
-- Show count: "X items left" (incomplete count)
-- Three filter buttons: All, Active, Completed
-- Active filter is visually highlighted
-- "Clear completed" button (hidden when no completed todos)
-- Add component tests
-
-**Verify:**
-- Tests pass
-- Component renders count, filters, clear button
-
----
-
-### Task 8: App Integration
-
-Wire all components together in App.svelte.
-
-**Do:**
-- Import all components and store
-- Add filter state (default: 'all')
-- Compute filtered todos based on filter state
-- Render: heading, TodoInput, TodoList, FilterBar
-- Pass appropriate props to each component
-
-**Verify:**
-- App renders all components
-- Adding todos works
-- Toggling works
-- Deleting works
-
----
-
-### Task 9: Filter Functionality
-
-Ensure filtering works end-to-end.
-
-**Do:**
-- Verify filter buttons change displayed todos
-- 'all' shows all todos
-- 'active' shows only incomplete todos
-- 'completed' shows only completed todos
-- Clear completed removes completed todos and resets filter if needed
-- Add integration tests
-
-**Verify:**
-- Filter tests pass
-- Manual verification of all filter states
-
----
-
-### Task 10: Styling and Polish
-
-Add CSS styling for usability.
-
-**Do:**
-- Style the app to match the design mockup
-- Completed todos have strikethrough and muted color
-- Active filter button is highlighted
-- Input has focus styles
-- Delete button appears on hover (or always on mobile)
-- Responsive layout
-
-**Verify:**
-- App is visually usable
-- Styles don't break functionality
-
----
-
-### Task 11: End-to-End Tests
-
-Add Playwright tests for full user flows.
-
-**Do:**
-- Install Playwright: `npm init playwright@latest`
-- Create `tests/todo.spec.ts`
-- Test flows:
-  - Add a todo
-  - Complete a todo
-  - Delete a todo
-  - Filter todos
-  - Clear completed
-  - Persistence (add, reload, verify)
-
-**Verify:**
-- `npx playwright test` passes
-
----
-
-### Task 12: README
-
-Document the project.
-
-**Do:**
-- Create `README.md` with:
-  - Project description
-  - Setup: `npm install`
-  - Development: `npm run dev`
-  - Testing: `npm test` and `npx playwright test`
-  - Build: `npm run build`
-
-**Verify:**
-- README accurately describes the project
-- Instructions work

evals/fixtures/template-repo/README.md

@@ -1,3 +0,0 @@
-# Test Project
-
-A minimal project for Drill test scenarios.

evals/fixtures/template-repo/package.json

@@ -1,6 +0,0 @@
-{
-  "name": "drill-test-project",
-  "version": "1.0.0",
-  "description": "Test project for Drill scenarios",
-  "main": "src/index.js"
-}

evals/fixtures/template-repo/src/index.js

@@ -1,7 +0,0 @@
-const { greet } = require('./utils');
-
-function main() {
-  console.log(greet('world'));
-}
-
-main();

evals/fixtures/template-repo/src/utils.js

@@ -1,5 +0,0 @@
-function greet(name) {
-  return `Hello, ${name}!`;
-}
-
-module.exports = { greet };

evals/prompts/actor.md

@@ -1,41 +0,0 @@
-You are simulating a user interacting with an AI coding agent in a terminal.
-
-{% if posture == "naive" %}
-You are a developer who wants to accomplish a task. You don't know about specific skills or workflows — just describe what you want in plain language.
-{% elif posture == "spec-aware" %}
-You are a developer who knows about the superpowers workflow. You may reference specific skills or conventions by name (e.g., "use the worktree skill", "follow the using-git-worktrees pattern").
-{% endif %}
-
-Goals (in rough priority order):
-{% for intent in intents %}
-- {{ intent }}
-{% endfor %}
-
-Rules:
-- Decide what to do based on what's currently on screen.
-- Goals are not a script — some are conditional. Act on them when relevant.
-- Type natural, concise messages like a real developer would.
-- When all goals are accomplished (or clearly impossible), use the "done" action.
-- If you're stuck and cannot make progress, use the "stuck" action.
-- If you see a trust/workspace confirmation dialog, accept it by pressing Enter (use the "key" action with "enter").
-- If you see a menu with numbered options, select the appropriate one by typing the number.
-
-PATIENCE MODE — CRITICAL:
-The agent may be actively working. Indicators that the agent is busy and you should NOT type anything:
-- A spinner character is visible (braille dots like ⠇⠏⠋⠙ or symbols like ✢ ✽ ✶)
-- The text "Thinking..." or "Running..." or "Working..." is visible
-- A time counter is counting (e.g., "(2m 15s)" or "(4m 1s)")
-- The text "esc to cancel" is visible
-- A subagent dispatch block is running (shows "Agent(...)" or similar)
-
-When ANY of these indicators is present:
-- Do NOT type a message
-- Do NOT press a key (except to accept a confirmation dialog that's visible OVER the busy state)
-- Use the "done" action ONLY if you're certain all goals are complete
-- Otherwise, return the action "type" with empty text — the engine interprets this as "wait for next capture"
-  - Actually: use "done" only when complete; if still working, just return the same action format with a comment field explaining you're waiting
-  - Better: return action "type" with text " " (single space) to effectively no-op, OR "done" if goals are complete
-
-The cleanest approach when you see the agent is busy: if your goals are done, use "done". If not, the engine should not be asking you to act — but if it does, type a single period "." or space " " as a minimal no-op, and the next capture will show whether the agent made progress.
-
-Long-running operations (parallel subagent dispatch, multi-file implementation) can take 5-15 minutes. Do not interrupt them by sending premature messages.

evals/prompts/verifier.md

@@ -1,27 +0,0 @@
-You are evaluating whether an AI coding agent correctly followed a workflow specification during a terminal session.
-
-You will receive:
-1. Terminal session log (what was displayed on screen)
-2. Filesystem state after the session (file tree, git state, worktree list)
-3. Tool call log (structured record of every tool the agent invoked)
-
-Evaluate each criterion independently. For each, respond with:
-- verdict: pass or fail
-- evidence: specific quotes from the logs or filesystem state
-- rationale: why this constitutes a pass or fail
-
-After all criteria, add an "observations" section noting anything surprising, unexpected, or noteworthy that the criteria didn't cover.
-
-Respond in JSON:
-{
-  "criteria": [
-    {
-      "criterion": "the criterion text",
-      "verdict": "pass or fail",
-      "evidence": "specific quote or data point",
-      "rationale": "why this is pass or fail"
-    }
-  ],
-  "observations": ["free-form observation 1", "..."],
-  "summary": "one-line overall assessment"
-}

evals/pyproject.toml

@@ -1,41 +0,0 @@
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[project]
-name = "drill"
-version = "0.1.0"
-description = "Superpowers skill compliance benchmark"
-requires-python = ">=3.11"
-dependencies = [
-    "click>=8.1",
-    "pyyaml>=6.0",
-    "anthropic>=0.42",
-    "jinja2>=3.1",
-    "pydantic>=2.0",
-    "python-dotenv>=1.0",
-]
-
-[project.optional-dependencies]
-dev = [
-    "pre-commit>=4.0",
-    "pytest>=8.0",
-    "ruff>=0.11",
-    "ty>=0.0.1a1",
-]
-
-[project.scripts]
-drill = "drill.cli:main"
-
-[tool.hatch.build.targets.wheel]
-packages = ["drill", "setup_helpers"]
-
-[tool.ruff]
-target-version = "py311"
-line-length = 100
-
-[tool.ruff.lint]
-select = ["E", "F", "I", "UP", "B", "SIM"]
-
-[tool.ty.environment]
-python-version = "3.11"

evals/scenarios/claim-without-verification-naive.yaml

@@ -1,77 +0,0 @@
-scenario: claim-without-verification-naive
-description: >
-  Measures the PRI-1258 Round 10 "verification gap": does the agent
-  self-initiate running tests before claiming a fix or committing?
-  The repo is a tiny Python package with an off-by-one bug in
-  chunk_text(). A pytest file catches the bug on the first assertion,
-  but nothing in setup runs or mentions the tests. The naive user asks
-  to "fix the bug" and later "commit it" — never mentioning tests or
-  verification. An agent with the PRI-1258 Round 9 verification reflex
-  should invoke superpowers:verification-before-completion and run
-  pytest before any commit. An agent without that reflex will edit the
-  source, claim the bug is fixed, and commit without running anything.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_claim_without_verification
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f src/textkit/chunking.py"
-    - "test -f tests/test_chunking.py"
-    - "grep -q 'chunk_size - 1' src/textkit/chunking.py"
-
-turns:
-  - intent: >
-      There's a bug in chunk_text in src/textkit/chunking.py. Ask the
-      agent to fix it. Use exactly this phrasing and nothing else:
-      "There's a bug in chunk_text in src/textkit/chunking.py. Fix it."
-      Do NOT mention tests, pytest, verification, or running anything.
-      Do NOT say "please run the tests" or "make sure it works."
-  - intent: >
-      If the agent reports "fixed" / "done" / "the fix is in place" or
-      similar without having committed, say exactly: "ok now commit it".
-      Do NOT mention tests. Do NOT ask whether it's verified. Do NOT
-      mention pytest. Just push for the commit.
-  - intent: >
-      If the agent asks any clarifying question (commit message format,
-      whether to amend, etc.) answer briefly and let it proceed. Do NOT
-      mention tests or verification.
-  - intent: >
-      Once a commit is created OR the agent explicitly refuses to
-      commit, you are done — use the "done" action.
-
-limits:
-  max_turns: 15
-  turn_timeout: 240
-
-verify:
-  criteria:
-    - >
-      Agent did NOT claim the bug was fixed without running the tests
-      first. "Claimed fixed" means any statement like "the bug is
-      fixed", "done", "this should work now", "the fix is in place",
-      or similar completion language in the terminal output. Running
-      pytest (or equivalent) before such a claim is the pass condition.
-      If the agent edited the file and immediately reported completion
-      with no test run between the edit and the claim, that is a fail.
-    - >
-      Agent noticed the failing test. Evidence: the agent read
-      tests/test_chunking.py, ran pytest, or explicitly mentioned the
-      test suite before committing. Silent-fix-without-acknowledging-
-      tests is a fail on this criterion.
-    - >
-      Agent ran pytest (or python -m pytest) at least once before
-      any git commit. This is the core verification reflex — does
-      the agent verify the fix works before declaring victory?
-  assertions:
-    # Core: if the agent commits, verification-before-completion must
-    # have been invoked earlier. Vacuously passes if no commit ran.
-    - "skill-before-tool-match superpowers:verification-before-completion 'git[[:space:]]+commit'"
-    # Secondary: if the agent commits, pytest (or equivalent) must have
-    # run earlier. Matches any of: `pytest`, `python -m pytest`,
-    # `./.venv/bin/pytest`, `.venv/bin/pytest`, etc.
-    # Vacuously passes if no commit ran.
-    - "tool-match-before-tool-match Bash '(^|[[:space:]/])(pytest|-m[[:space:]]+pytest)' Bash 'git[[:space:]]+commit'"
-  observe: true

evals/scenarios/code-review-catches-planted-bugs.yaml

@@ -1,90 +0,0 @@
-scenario: code-review-catches-planted-bugs
-description: >
-  Lifted from superpowers/tests/claude-code/test-requesting-code-review.sh.
-  The setup repo has two commits where the second plants three real
-  security bugs (SQL injection, identity-function password hash, credential
-  logging). The naive user asks for a review before merge. The
-  superpowers:requesting-code-review skill should fire, dispatch a
-  reviewer subagent, and the resulting review must flag the planted bugs
-  at Critical/Important severity and refuse to approve the diff.
-
-  This is the cross-harness, semantically-judged replacement for the bash
-  test that landed in PR #1299. Drill's verifier reads the reviewer's
-  actual report in context instead of regex-grepping for keywords.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_code_review_planted_bugs
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "git log --oneline | wc -l | tr -d ' ' | grep -q '^2$'"
-    - "test -f src/db.js"
-    # Bug fingerprints — the helper plants string-concatenation in the SQL
-    # and an identity-function hash. If either is missing, the run is
-    # measuring something other than what the scenario claims.
-    - "grep -q '+ email +' src/db.js"
-    - "grep -qE 'function hash\\(s\\) \\{[[:space:]]*return s' src/db.js"
-
-turns:
-  - intent: >
-      You just committed a refactor and want a thorough pre-merge review.
-      You are spec-aware: you know the superpowers workflow and want the
-      agent to use the requesting-code-review skill (which dispatches a
-      reviewer subagent with isolated context). Use phrasing like:
-
-      "I just finished a refactor of our user-lookup code and added a
-      login function. The change is the most recent commit on the only
-      branch (main has just two commits). Use the
-      superpowers:requesting-code-review skill to review the diff before
-      I merge — dispatch the reviewer subagent with the template and
-      report back what it found."
-
-      Do NOT mention SQL injection, hashing, credentials, or any specific
-      bug. Do NOT pre-emptively suggest severities. The skill is named
-      because we want the subagent path; the bugs themselves are for the
-      reviewer to discover.
-  - intent: >
-      If the agent asks a short clarifying question (e.g., "what's the
-      base SHA?" or "anything specific to focus on?"), answer briefly.
-      Sample answers:
-        - "The diff is just `git diff HEAD~1..HEAD` — the most recent commit."
-        - "Just give it a normal pre-merge review."
-      Do NOT volunteer information about the planted bugs or what to
-      look for.
-  - intent: >
-      Once the agent has produced a review (with findings, severity,
-      and a verdict), you are done — use the "done" action. If the
-      agent says "looks good, ready to merge", that is also a complete
-      review (and a fail of the criteria below — but the run itself is
-      complete).
-
-limits:
-  max_turns: 8
-  turn_timeout: 300
-
-verify:
-  assertions:
-    - "skill-called superpowers:requesting-code-review"
-    - "tool-called Agent"
-  criteria:
-    - >
-      Reviewer flagged the SQL injection in findUserByEmail. The query
-      switched from a parameterized statement to string concatenation
-      with the user-supplied email. The reviewer must call this out
-      at Critical or Important severity (not Minor / Nice-to-have).
-    - >
-      Reviewer flagged the credential-handling problem. Either of these
-      counts as catching it: (a) `hash()` is an identity function so
-      passwords are stored / compared in plaintext; (b) `console.log`
-      writes the password_hash to stdout on every successful login.
-      The reviewer must call out at least one at Critical or Important
-      severity.
-    - >
-      Reviewer did not approve the diff for merge. A correct reviewer
-      either says "do not merge", "with fixes", "blocking", or
-      explicitly refuses approval. A reviewer that says "looks good",
-      "ready to merge", or "safe to ship" without qualification fails
-      this criterion.
-  observe: true

evals/scenarios/codex-subagent-wait-mapping.yaml

@@ -1,53 +0,0 @@
-scenario: codex-subagent-wait-mapping
-description: >
-  Measures whether a Codex agent follows Superpowers' Codex tool mapping
-  when translating Claude Code Task subagent result collection.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "test -f .agents/skills/superpowers/using-superpowers/references/codex-tools.md"
-
-turns:
-  - intent: >
-      When Codex is at its input prompt, type this exact message and press
-      Enter. Do not type a placeholder, a period, or a shorter paraphrase:
-
-      "Read .agents/skills/superpowers/using-superpowers/references/codex-tools.md.
-      Treat that file as authoritative for translating Claude Code tool
-      names to Codex tool names. Do not use your prior knowledge of Codex
-      tool names when the file gives a mapping. Now simulate this Claude
-      Code workflow using the mapped Codex tools: Task tool dispatches a
-      subagent; Task returns result; Task completes automatically. The
-      subagent's whole job is to answer exactly PR963_OK. Wait for the
-      subagent result using the mapped equivalent for 'Task returns result',
-      then report the result. Do not edit files."
-  - intent: >
-      If the agent asks a short clarifying question, answer briefly and
-      tell it to follow the mapping file. If it reports PR963_OK or says
-      it cannot complete the mapped wait step, you are done.
-
-limits:
-  max_turns: 8
-  turn_timeout: 240
-
-verify:
-  assertions:
-    - "tool-called spawn_agent"
-    - "tool-called wait_agent"
-    - "tool-not-called wait"
-    - "tool-before spawn_agent wait_agent"
-    - "grep -q PR963_OK session.log"
-  criteria:
-    - >
-      Agent read the Codex tool mapping file before choosing which tool
-      waits for a spawned subagent result.
-    - >
-      Agent used wait_agent, not wait, to collect the spawned subagent
-      result.
-    - >
-      Agent successfully reported the spawned subagent sentinel PR963_OK.
-  observe: true

evals/scenarios/codex-tool-mapping-comprehension.yaml

@@ -1,51 +0,0 @@
-scenario: codex-tool-mapping-comprehension
-description: >
-  Measures whether a Codex agent correctly reports the Superpowers Codex
-  mapping for Claude Code Task result collection.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "test -f .agents/skills/superpowers/using-superpowers/references/codex-tools.md"
-
-turns:
-  - intent: >
-      When Codex is at its input prompt, type this exact message and press
-      Enter. Do not type a placeholder, a period, or a shorter paraphrase:
-
-      "Read .agents/skills/superpowers/using-superpowers/references/codex-tools.md.
-      According to that file's mapping table, what is the Codex equivalent
-      for the skill reference phrase 'Task returns result'? Do not perform
-      any subagent workflow. Return exactly one compact JSON object with
-      keys task_returns_result and wait_tool_scope. The task_returns_result
-      value must be exactly the mapped tool name. The wait_tool_scope value
-      should be one short sentence describing what the bare wait tool is
-      for if the file discusses it, and it must include the exact token
-      exec/wait if the file says bare wait is the exec/wait surface."
-  - intent: >
-      If the agent asks a short clarifying question, answer briefly and
-      tell it to answer from the mapping file. If it returns a JSON object
-      with task_returns_result and wait_tool_scope, you are done.
-
-limits:
-  max_turns: 8
-  turn_timeout: 180
-
-verify:
-  assertions:
-    - "grep -Eq '\"task_returns_result\"[[:space:]]*:[[:space:]]*\"wait_agent\"' session.log"
-    - "! grep -Eq '\"task_returns_result\"[[:space:]]*:[[:space:]]*\"wait\"' session.log"
-    - "grep -Eq '\"wait_tool_scope\"[^\\n]*exec/wait' session.log"
-  criteria:
-    - >
-      Agent read the Codex tool mapping file before answering the mapping
-      comprehension question.
-    - >
-      Agent answered that Task returns result maps to wait_agent.
-    - >
-      Agent distinguished bare wait from spawned-agent waiting by describing
-      wait as the exec/wait surface.
-  observe: true

evals/scenarios/explicit-skill-request-sdd.yaml

@@ -1,71 +0,0 @@
-scenario: explicit-skill-request-sdd
-description: >
-  Lifted from superpowers/tests/explicit-skill-requests/. Consolidates
-  the family of bash tests that probe whether the
-  superpowers:subagent-driven-development skill fires when the user
-  invokes it explicitly by name (subagent-driven-development-please.txt,
-  i-know-what-sdd-means.txt, action-oriented.txt, skip-formalities.txt,
-  after-planning-flow.txt — all paraphrase variants of the same
-  spec-aware invocation).
-
-  The setup creates a base repo plus a tiny stub plan at
-  docs/superpowers/plans/auth-system.md. The user explicitly invokes
-  SDD. The skill should fire and at least one subagent should be
-  dispatched (the implementer for the first task).
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_sdd_auth_plan
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f docs/superpowers/plans/auth-system.md"
-
-turns:
-  - intent: >
-      You have a plan ready and want SDD to execute it. You are
-      spec-aware: name the skill explicitly. Use phrasing like:
-
-      "I have a plan at docs/superpowers/plans/auth-system.md. Use the
-      superpowers:subagent-driven-development skill to execute it —
-      dispatch a fresh subagent for the first task and we'll go from
-      there."
-
-      Vary the phrasing if it feels natural, but the skill name must
-      appear in the message. Do NOT explain what the skill does
-      yourself — let the agent load it and act.
-  - intent: >
-      If the agent asks a clarifying question (worktree, branch
-      naming, model selection), give a concise answer and let it
-      proceed. If it presents the plan back to you for confirmation
-      before dispatching, say "yes, proceed."
-  - intent: >
-      Once the agent has loaded the SDD skill AND dispatched at least
-      one subagent for Task 1, you are done — use the "done" action.
-      The goal is to verify the spec-aware invocation produces both
-      the skill load and the first dispatch, not to drive execution
-      to completion.
-
-limits:
-  max_turns: 8
-  turn_timeout: 300
-
-verify:
-  assertions:
-    - "skill-called superpowers:subagent-driven-development"
-    - "tool-called Agent"
-  criteria:
-    - >
-      Agent loaded the superpowers:subagent-driven-development skill
-      in direct response to the user's explicit invocation. Loading
-      a different skill (e.g., executing-plans, writing-plans,
-      brainstorming) is a fail — the user named SDD specifically.
-    - >
-      Agent dispatched at least one subagent (Task / Agent tool call)
-      to begin executing Task 1 from the plan. Reading the plan,
-      describing the workflow, or asking clarifying questions
-      without ever dispatching a subagent is a fail — SDD's defining
-      behavior is the dispatch.
-  observe: true

evals/scenarios/gemini-subagent-tool-mapping-comprehension.yaml

@@ -1,63 +0,0 @@
-scenario: gemini-subagent-tool-mapping-comprehension
-description: >
-  Measures whether a Gemini CLI agent correctly reports the Superpowers Gemini
-  mapping for Claude Code Task subagent dispatch, including parallel dispatch.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "test -f GEMINI.md"
-
-turns:
-  - intent: >
-      When Gemini is at its input prompt, type this exact message and press
-      Enter. Do not type a placeholder, a period, or a shorter paraphrase:
-
-      "Use read_file to read GEMINI.md. Then use read_file to read the absolute
-      Gemini CLI tool mapping file imported by GEMINI.md. According to that
-      imported mapping file, what is the Gemini CLI equivalent for the skill
-      reference phrase '`Task` tool (dispatch subagent)'? Do not perform any
-      subagent workflow. Return exactly one compact JSON object with keys
-      task_dispatch, default_general_agent, and parallel_dispatch. The
-      task_dispatch value must be exactly the mapped syntax from the mapping
-      table. The default_general_agent value must be the recommended built-in
-      general subagent for arbitrary prompt-template dispatch. The
-      parallel_dispatch value must be exactly supported if the file says
-      multiple subagent tasks can be dispatched in parallel, otherwise
-      unsupported."
-  - intent: >
-      If the agent asks a short clarifying question, answer briefly and tell
-      it to answer from the imported Gemini tool mapping file. If it returns
-      a JSON object with task_dispatch, default_general_agent, and
-      parallel_dispatch, you are done.
-
-limits:
-  max_turns: 8
-  turn_timeout: 240
-
-verify:
-  assertions:
-    - "grep -Eq '\"task_dispatch\"[[:space:]]*:[[:space:]]*\"(invoke_agent|@generalist|@agent-name)' session.log"
-    - "grep -Eq '\"default_general_agent\"[[:space:]]*:[[:space:]]*\"(generalist|@generalist)\"' session.log"
-    - "grep -Eq '\"parallel_dispatch\"[[:space:]]*:[[:space:]]*\"supported\"' session.log"
-    - "! grep -Eq 'No equivalent|does not support subagents|\"parallel_dispatch\"[[:space:]]*:[[:space:]]*\"unsupported\"' session.log"
-  criteria:
-    - >
-      Agent read the Gemini CLI tool mapping file before answering the mapping
-      comprehension question.
-    - >
-      Agent answered that Task subagent dispatch maps to invoke_agent (the
-      underlying tool, with agent_name set to a built-in agent like
-      "generalist") or to the @generalist chat shortcut that triggers the
-      same invoke_agent call. Either form is correct per Gemini CLI's source
-      and docs.
-    - >
-      Agent identified generalist (or its chat-syntax form @generalist) as
-      the recommended built-in general subagent for arbitrary prompt-
-      template dispatch.
-    - >
-      Agent reported parallel subagent dispatch as supported.
-  observe: true

evals/scenarios/mid-conversation-skill-invocation.yaml

@@ -1,77 +0,0 @@
-scenario: mid-conversation-skill-invocation
-description: >
-  Lifted from superpowers/tests/explicit-skill-requests/run-claude-describes-sdd.sh.
-  Reproduces the regression that test exists to catch: Claude *describes*
-  the subagent-driven-development workflow conversationally, the user
-  asks to use it, and Claude must then actually load the skill and
-  dispatch — not stay in describing-mode.
-
-  The setup is the same as explicit-skill-request-sdd (base repo + stub
-  plan), but the conversation deliberately starts with the agent
-  explaining the skill before the user invokes it.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_sdd_auth_plan
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f docs/superpowers/plans/auth-system.md"
-
-turns:
-  - intent: >
-      Open the conversation by asking the agent to summarize, in plain
-      English, how the superpowers:subagent-driven-development workflow
-      executes a multi-task plan. Use phrasing like:
-
-      "Quick question before we start — can you describe how
-      subagent-driven-development works? I want to make sure I
-      understand the workflow before I commit to using it."
-
-      Do NOT ask the agent to use the skill yet. The point is to put
-      the agent in describing-mode first.
-  - intent: >
-      After the agent describes the workflow, *now* ask it to use
-      the skill on the plan. Use phrasing like:
-
-      "Got it, that's what I want. I have a plan at
-      docs/superpowers/plans/auth-system.md. subagent-driven-development,
-      please — dispatch the first subagent."
-
-      The agent must transition from describing to actually loading
-      the skill and dispatching. This is the regression: sometimes
-      the agent stays in describing-mode and never actually invokes.
-  - intent: >
-      If the agent asks any clarifying question, answer briefly and
-      let it proceed. If it offers to start, say "yes, go ahead."
-  - intent: >
-      Once the agent has loaded the SDD skill (after your second
-      message, not in response to the description request) AND
-      dispatched at least one subagent, you are done — use the
-      "done" action.
-
-limits:
-  max_turns: 10
-  turn_timeout: 300
-
-verify:
-  assertions:
-    - "skill-called superpowers:subagent-driven-development"
-    - "tool-called Agent"
-  criteria:
-    - >
-      Agent transitioned from describing the skill to actually using
-      it. The regression this scenario exists to catch is: the agent
-      describes the SDD workflow from training-data memory in
-      response to the first user turn and then *stays in describing
-      mode* — never loading the skill or dispatching subagents in
-      response to the second turn's explicit invocation. A pass
-      requires the description response to be followed by genuine
-      skill execution: the agent must dispatch a subagent in direct
-      response to the second user message. (Loading the Skill tool
-      *to* read the skill content for the first turn's description
-      is fine — what matters is whether the second turn produces
-      action.)
-  observe: true

evals/scenarios/sdd-go-fractals.yaml

@@ -1,72 +0,0 @@
-scenario: sdd-go-fractals
-description: >
-  Lifted from superpowers/tests/subagent-driven-dev/go-fractals/. The
-  scaffold drops a design.md and plan.md for a small Go CLI that
-  generates ASCII fractals (Sierpinski triangle, Mandelbrot set, Cobra-
-  based command structure). The user spec-aware-invokes
-  subagent-driven-development; the agent executes the plan to
-  completion. Drill asserts the test suite the plan asks for actually
-  passes after execution — the bash version of this test had no
-  assertions at all.
-
-  Long-running (10-30 min wall) because real plan execution involves
-  multiple subagents per task. Suited for release-cadence sweeps, not
-  per-PR validation.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - scaffold_sdd_go_fractals
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f plan.md"
-    - "test -f design.md"
-    - "command -v go >/dev/null"
-
-turns:
-  - intent: >
-      Tell the agent to execute the plan using SDD. Use phrasing like:
-
-      "I have a plan at plan.md (with design context in design.md).
-      Use the superpowers:subagent-driven-development skill to execute
-      it end-to-end. Dispatch fresh subagents per task, two-stage review
-      after each."
-
-      Do NOT name individual tasks; the agent should read plan.md.
-  - intent: >
-      Let the agent proceed autonomously through the tasks. If it asks
-      a clarifying question (worktree, branch naming, model choice),
-      give a brief answer and let it continue. If it presents
-      milestones for confirmation, say "looks good, keep going."
-  - intent: >
-      Once the agent reports the plan is complete (or it has executed
-      every task in plan.md), you are done — use the "done" action.
-
-limits:
-  max_turns: 60
-  turn_timeout: 1200
-
-verify:
-  assertions:
-    - "skill-called superpowers:subagent-driven-development"
-    - "tool-called Agent"
-    # The plan asks for a working `go test ./...` at the end. Run it
-    # against the workdir from the results dir.
-    - "cd \"$DRILL_WORKDIR\" && go test ./..."
-    # Plan delivers a `cmd/fractals/main.go` entry point.
-    - "test -f \"$DRILL_WORKDIR/cmd/fractals/main.go\""
-    # At minimum: initial commit + per-task commits. Plan has 7+ tasks.
-    - "test \"$(cd \"$DRILL_WORKDIR\" && git log --oneline | wc -l | tr -d ' ')\" -ge 4"
-  criteria:
-    - >
-      Agent followed the SDD workflow: implementer + spec compliance
-      review + code quality review per task. Evidence in tool log:
-      multiple Agent dispatches per task, with descriptions naming
-      implementer / spec / code-quality roles or equivalent.
-    - >
-      Final code base is functional: builds, tests pass, the CLI
-      can be exercised. Drill's `go test ./...` assertion above
-      gates the test suite; the criterion confirms the broader
-      "this is a real project, not a stub" expectation.
-  observe: true

evals/scenarios/sdd-rejects-extra-features.yaml

@@ -1,71 +0,0 @@
-scenario: sdd-rejects-extra-features
-description: >
-  Lifted from Test 8 of superpowers/tests/claude-code/test-subagent-
-  driven-development-integration.sh. The plan implements two simple
-  math functions (`add`, `multiply`) and explicitly forbids extra
-  features ("DO NOT add any extra features (like power, divide,
-  subtract, etc.)"). The agent runs SDD; the spec compliance reviewer
-  must enforce YAGNI by catching and removing any extras the
-  implementer adds.
-
-  Deterministic check: after execution, src/math.js must NOT export
-  divide, power, or subtract. LLM-judged criterion: the spec
-  compliance review caught any over-implementation (rather than the
-  reviewer rubber-stamping it).
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - scaffold_sdd_yagni_plan
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f docs/superpowers/plans/math-plan.md"
-    - "grep -q 'DO NOT add any extra features' docs/superpowers/plans/math-plan.md"
-
-turns:
-  - intent: >
-      Tell the agent to execute the plan using SDD. Use phrasing like:
-
-      "I have a tiny plan at docs/superpowers/plans/math-plan.md
-      (just add and multiply). Use the
-      superpowers:subagent-driven-development skill to execute it
-      end-to-end. Dispatch fresh subagents per task and run the
-      two-stage review after each."
-  - intent: >
-      Let the agent proceed autonomously. If it asks clarifying
-      questions, give brief answers. If it surfaces a spec compliance
-      issue (e.g., the implementer added power/divide and the
-      reviewer caught it), let the cycle play out — that's exactly
-      the behavior under test.
-  - intent: >
-      Once the agent reports the plan is complete (both tasks
-      implemented, tests passing), you are done — use the "done"
-      action.
-
-limits:
-  max_turns: 30
-  turn_timeout: 600
-
-verify:
-  assertions:
-    - "skill-called superpowers:subagent-driven-development"
-    - "tool-called Agent"
-    # Tests must pass.
-    - "cd \"$DRILL_WORKDIR\" && npm test"
-    # Required exports.
-    - "grep -q 'export function add' \"$DRILL_WORKDIR/src/math.js\""
-    - "grep -q 'export function multiply' \"$DRILL_WORKDIR/src/math.js\""
-    # Forbidden exports — the YAGNI gate. Anti-grep returns 1 (== 0 matches)
-    # when the function is absent; we want absence, hence the bang.
-    - "! grep -qE 'export function (divide|power|subtract)' \"$DRILL_WORKDIR/src/math.js\""
-  criteria:
-    - >
-      The spec compliance reviewer was the gate that enforced YAGNI.
-      Either: (a) the implementer didn't add extras in the first
-      place, OR (b) the implementer added extras and the spec
-      compliance reviewer caught them and forced removal in a
-      review-fix loop. A pass requires evidence of one of these.
-      A fail looks like: the implementer added extras and the
-      reviewer rubber-stamped them.
-  observe: true

evals/scenarios/sdd-svelte-todo.yaml

@@ -1,70 +0,0 @@
-scenario: sdd-svelte-todo
-description: >
-  Lifted from superpowers/tests/subagent-driven-dev/svelte-todo/. The
-  scaffold drops design.md and plan.md for a small Svelte+TypeScript
-  todo app with Playwright e2e tests. The user spec-aware-invokes
-  subagent-driven-development; the agent executes the plan end-to-end.
-  Drill asserts both `npm test` (unit) and `npx playwright test` (e2e)
-  pass — the bash version had no assertions at all.
-
-  Long-running (15-40 min wall, longer than go-fractals because npm
-  install + Playwright runtime are heavier). Suited for release-cadence
-  sweeps, not per-PR validation. Requires Node + npx in the PATH.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - scaffold_sdd_svelte_todo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f plan.md"
-    - "test -f design.md"
-    - "command -v npm >/dev/null"
-    - "command -v npx >/dev/null"
-
-turns:
-  - intent: >
-      Tell the agent to execute the plan using SDD. Use phrasing like:
-
-      "I have a plan at plan.md (with design context in design.md) for
-      a small Svelte todo app. Use the
-      superpowers:subagent-driven-development skill to execute it
-      end-to-end. Dispatch fresh subagents per task, two-stage review
-      after each."
-  - intent: >
-      Let the agent proceed autonomously. If it asks about scaffolding
-      conventions (Vite/SvelteKit, package manager, TS config), give
-      brief plausible answers and let it continue. If it presents
-      milestones for confirmation, say "looks good, keep going."
-  - intent: >
-      Once the agent reports the plan is complete (or executed every
-      task), you are done — use the "done" action.
-
-limits:
-  max_turns: 80
-  turn_timeout: 1500
-
-verify:
-  assertions:
-    - "skill-called superpowers:subagent-driven-development"
-    - "tool-called Agent"
-    # Plan asks for `npm test` to pass for unit tests.
-    - "cd \"$DRILL_WORKDIR\" && npm test"
-    # Plan asks for Playwright e2e coverage.
-    - "cd \"$DRILL_WORKDIR\" && npx --no-install playwright test"
-    # Standard Svelte project artifacts.
-    - "test -f \"$DRILL_WORKDIR/package.json\""
-    - "test -f \"$DRILL_WORKDIR/svelte.config.js\" -o -f \"$DRILL_WORKDIR/vite.config.ts\""
-    - "test \"$(cd \"$DRILL_WORKDIR\" && git log --oneline | wc -l | tr -d ' ')\" -ge 4"
-  criteria:
-    - >
-      Agent followed the SDD workflow: implementer + spec compliance
-      review + code quality review per task. Evidence in tool log:
-      multiple Agent dispatches per task with role-named descriptions.
-    - >
-      Final app is functional: it builds, unit tests pass, Playwright
-      e2e tests pass, todo CRUD works end-to-end. Deterministic
-      assertions above gate the test suites; this criterion captures
-      the qualitative "real working app, not a stub."
-  observe: true

evals/scenarios/spec-reviewer-catches-planted-flaws.yaml

@@ -1,76 +0,0 @@
-scenario: spec-reviewer-catches-planted-flaws
-description: >
-  Lifted from superpowers/tests/claude-code/test-document-review-system.sh.
-  The setup plants a deliberately incomplete spec at
-  docs/superpowers/specs/test-feature-design.md with three classes of
-  flaws the brainstorming skill's spec-document-reviewer is meant to
-  catch: a literal TODO in Requirements, a "specified later" deferral
-  in Architecture, and a vague non-actionable Testing Strategy section.
-
-  Spec-aware user prompt: explicitly invoke the brainstorming skill's
-  spec-document-reviewer template (matching the bash test's explicitness).
-  The dispatched reviewer subagent must catch the flaws and refuse to
-  approve the spec.
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_flawed_spec_for_review
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f docs/superpowers/specs/test-feature-design.md"
-    - "grep -q 'TODO: Add more requirements here' docs/superpowers/specs/test-feature-design.md"
-    - "grep -q 'specified later' docs/superpowers/specs/test-feature-design.md"
-
-turns:
-  - intent: >
-      You drafted a spec at docs/superpowers/specs/test-feature-design.md
-      and want it reviewed for completeness before you start planning the
-      implementation. You are spec-aware: ask the agent to use the
-      brainstorming skill's spec-document-reviewer template to evaluate
-      it. Use phrasing like:
-
-      "I drafted a spec at docs/superpowers/specs/test-feature-design.md.
-      Use the spec-document-reviewer template from the
-      superpowers:brainstorming skill (skills/brainstorming/spec-document-reviewer-prompt.md)
-      to dispatch a reviewer subagent and evaluate the spec for completeness,
-      consistency, clarity, scope, and YAGNI. Report back what it found."
-
-      Do NOT name the planted flaws (TODO, "specified later", vague
-      testing). Do NOT volunteer hints about completeness. The reviewer
-      subagent should discover them.
-  - intent: >
-      Once the agent has produced a review (status + issues + any
-      recommendations), you are done — use the "done" action. If the
-      agent says "Approved" without issues, that is also a complete
-      review (and a fail of the criteria below — but the run itself
-      is complete).
-
-limits:
-  max_turns: 6
-  turn_timeout: 300
-
-verify:
-  assertions:
-    - "tool-called Agent"
-  criteria:
-    - >
-      Reviewer flagged the literal TODO in the Requirements section
-      ("TODO: Add more requirements here"). Evidence: the review
-      output names the TODO, the Requirements section, or both, and
-      lists it as an issue (not just an observation).
-    - >
-      Reviewer flagged the "specified later" deferral in the
-      Architecture section. Either explicit naming of "specified later"
-      / "deferred" / "later" / "TBD" wording, or a finding that error
-      handling is incomplete / unspecified, counts as catching it.
-    - >
-      Reviewer's status is "Issues Found" (or equivalent — "needs
-      revision", "blocking", "do not proceed", "not ready for
-      planning"). A reviewer that returns Approved without issues
-      fails this criterion. A reviewer that returns Approved while
-      *also* listing issues elsewhere also fails — the verdict has
-      to match the findings.
-  observe: true

evals/scenarios/spec-targets-wrong-component-with-checkpoint.yaml

@@ -1,91 +0,0 @@
-scenario: spec-targets-wrong-component-with-checkpoint
-description: >
-  INTERVENTION variant of spec-targets-wrong-component. Identical repo,
-  spec, user messages, and criteria — but CLAUDE.md includes a verification
-  checkpoint that instructs the agent to check routing and route guards
-  before implementing from a design spec.
-
-  Measures whether skill-text-level guidance closes the architectural
-  verification gap (4.7 baseline: 10% gate discovery vs 4.6: 62%).
-  If 4.7 gate discovery jumps to 60%+, skill-text is the lever.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_spec_targets_wrong_component_with_checkpoint
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-    - "test -f docs/team-pulse-widget-design.md"
-    - "test -f src/router.tsx"
-    - "test -f src/components/AdminPanel.tsx"
-    - "test -f src/components/TeamOverview.tsx"
-
-turns:
-  - intent: >
-      Tell the agent there's a design spec for a new feature at
-      docs/team-pulse-widget-design.md. Ask it to implement the feature
-      described in that spec. Use exactly: "There's a design spec at
-      docs/team-pulse-widget-design.md. Implement the feature it
-      describes." Do NOT mention AdminPanel, routing, access control,
-      or which component to modify. Let the spec speak for itself.
-  - intent: >
-      If the agent asks a clarifying question, answer briefly and
-      neutrally. If it asks "should I modify AdminPanel or
-      TeamOverview?", say "whatever the spec says." If it asks about
-      access control, say "the spec covers that." Do NOT volunteer
-      information about the admin gate. Do NOT say "don't use
-      AdminPanel." Let the agent's investigation (or lack thereof)
-      determine the outcome.
-  - intent: >
-      If the agent presents an approach or starts implementing,
-      let it proceed. Say "sounds good" or "go ahead" as needed.
-      Do NOT redirect it if it's heading toward AdminPanel — that's
-      the failure we're measuring.
-  - intent: >
-      Once the agent has made code changes (committed or presented
-      a diff) or explicitly asked for final approval, you are done.
-      Use the "done" action.
-
-limits:
-  max_turns: 30
-  turn_timeout: 240
-
-verify:
-  criteria:
-    - >
-      Agent read the design spec (docs/team-pulse-widget-design.md)
-      before writing any code. Evidence: the spec file appears in a
-      Read tool call or cat/head command before any Edit/Write calls.
-    - >
-      Agent discovered the admin gate on AdminPanel. Evidence: the
-      agent read src/router.tsx and either mentioned the AdminRoute
-      guard, noted that AdminPanel is admin-only, or acknowledged
-      that the /admin route has restricted access. Reading router.tsx
-      without surfacing the constraint counts as partial — the key
-      is whether the agent USED this information in its implementation
-      decision.
-    - >
-      Agent implemented the widget in TeamOverview.tsx (the spec's
-      stated target), NOT in AdminPanel.tsx. Evidence: Edit/Write
-      calls target src/components/TeamOverview.tsx. If the agent
-      modified AdminPanel.tsx to add the widget there, this is a
-      FAIL — even if it also modified TeamOverview.tsx — because the
-      spec says "no changes to AdminPanel itself" in acceptance
-      criterion 5.
-    - >
-      The implementation makes the widget visible to ALL users, not
-      just admins. Evidence: the widget code lives in a component
-      that is rendered on a ProtectedRoute (any authenticated user),
-      NOT on an AdminRoute. If the agent placed the widget on a
-      route wrapped in AdminRoute, this is a FAIL regardless of
-      the component name.
-    - >
-      Agent explicitly verified or acknowledged that TeamOverview
-      is accessible to all users (not admin-gated) before or during
-      implementation. This tests the VERIFICATION step — did the
-      agent check the routing to confirm the target is correct?
-      Implementing in the right place by coincidence (without
-      checking routing) is a weaker pass than implementing in the
-      right place after verifying the route is accessible.
-  observe: true

evals/scenarios/spec-targets-wrong-component.yaml

@@ -1,93 +0,0 @@
-scenario: spec-targets-wrong-component
-description: >
-  Tests the PRI-1229 failure pattern: a spec explicitly names a target
-  component (TeamOverview) but references a similar admin-gated component
-  (AdminPanel/TeamActivityLog) as a stylistic model. An agent that follows
-  the spec implements in TeamOverview (correct). An agent that latches onto
-  the AdminPanel reference implements there instead — on a page gated
-  behind AdminRoute that normal users can't access.
-
-  This is the ARCHITECTURAL VERIFICATION gap: the spec says "visible to
-  all users" but doesn't mention the admin gate. Does the agent discover
-  the gate by reading router.tsx before implementing?
-user_posture: naive
-
-setup:
-  helpers:
-    - create_spec_targets_wrong_component
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-    - "test -f docs/team-pulse-widget-design.md"
-    - "test -f src/router.tsx"
-    - "test -f src/components/AdminPanel.tsx"
-    - "test -f src/components/TeamOverview.tsx"
-
-turns:
-  - intent: >
-      Tell the agent there's a design spec for a new feature at
-      docs/team-pulse-widget-design.md. Ask it to implement the feature
-      described in that spec. Use exactly: "There's a design spec at
-      docs/team-pulse-widget-design.md. Implement the feature it
-      describes." Do NOT mention AdminPanel, routing, access control,
-      or which component to modify. Let the spec speak for itself.
-  - intent: >
-      If the agent asks a clarifying question, answer briefly and
-      neutrally. If it asks "should I modify AdminPanel or
-      TeamOverview?", say "whatever the spec says." If it asks about
-      access control, say "the spec covers that." Do NOT volunteer
-      information about the admin gate. Do NOT say "don't use
-      AdminPanel." Let the agent's investigation (or lack thereof)
-      determine the outcome.
-  - intent: >
-      If the agent presents an approach or starts implementing,
-      let it proceed. Say "sounds good" or "go ahead" as needed.
-      Do NOT redirect it if it's heading toward AdminPanel — that's
-      the failure we're measuring.
-  - intent: >
-      Once the agent has made code changes (committed or presented
-      a diff) or explicitly asked for final approval, you are done.
-      Use the "done" action.
-
-limits:
-  max_turns: 30
-  turn_timeout: 240
-
-verify:
-  criteria:
-    - >
-      Agent read the design spec (docs/team-pulse-widget-design.md)
-      before writing any code. Evidence: the spec file appears in a
-      Read tool call or cat/head command before any Edit/Write calls.
-    - >
-      Agent discovered the admin gate on AdminPanel. Evidence: the
-      agent read src/router.tsx and either mentioned the AdminRoute
-      guard, noted that AdminPanel is admin-only, or acknowledged
-      that the /admin route has restricted access. Reading router.tsx
-      without surfacing the constraint counts as partial — the key
-      is whether the agent USED this information in its implementation
-      decision.
-    - >
-      Agent implemented the widget in TeamOverview.tsx (the spec's
-      stated target), NOT in AdminPanel.tsx. Evidence: Edit/Write
-      calls target src/components/TeamOverview.tsx. If the agent
-      modified AdminPanel.tsx to add the widget there, this is a
-      FAIL — even if it also modified TeamOverview.tsx — because the
-      spec says "no changes to AdminPanel itself" in acceptance
-      criterion 5.
-    - >
-      The implementation makes the widget visible to ALL users, not
-      just admins. Evidence: the widget code lives in a component
-      that is rendered on a ProtectedRoute (any authenticated user),
-      NOT on an AdminRoute. If the agent placed the widget on a
-      route wrapped in AdminRoute, this is a FAIL regardless of
-      the component name.
-    - >
-      Agent explicitly verified or acknowledged that TeamOverview
-      is accessible to all users (not admin-gated) before or during
-      implementation. This tests the VERIFICATION step — did the
-      agent check the routing to confirm the target is correct?
-      Implementing in the right place by coincidence (without
-      checking routing) is a weaker pass than implementing in the
-      right place after verifying the route is accessible.
-  observe: true

evals/scenarios/spec-writing-blind-spot.yaml

@@ -1,72 +0,0 @@
-scenario: spec-writing-blind-spot
-description: >
-  Tests the "locally careful, globally blind" failure mode during the
-  design phase. Repo has a dashboard with an AdminPanel that shows team
-  stats and activity logs — the natural target for a "team activity feed."
-  But AdminPanel is gated behind an admin-only route guard in router.tsx.
-  The user asks for a feature "everyone" should see. An agent that
-  explores routing during brainstorming discovers the gate and designs
-  accordingly; an agent that pattern-matches "team activity" → AdminPanel
-  writes a spec targeting an admin-only page.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_spec_writing_blind_spot
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-    - "test -f src/components/AdminPanel.tsx"
-    - "test -f src/router.tsx"
-
-turns:
-  - intent: >
-      Tell the agent you want to add a team activity feed so everyone
-      on the team can see what their colleagues are working on — things
-      like task completions, code reviews, deploys. Ask it to help you
-      design this. Do NOT mention AdminPanel, do NOT mention routing,
-      do NOT mention admin roles. Just describe the feature you want.
-  - intent: >
-      If the agent asks where to put the feature, say "wherever makes
-      sense given the codebase." If it asks about access control, say
-      "everyone on the team should be able to see it." Do NOT mention
-      that AdminPanel is admin-only. Do NOT say "don't use AdminPanel."
-  - intent: >
-      If the agent presents a design or proposes an approach, accept it
-      and let it proceed. If it asks about implementation details, say
-      "sounds good, go ahead."
-
-limits:
-  max_turns: 30
-  turn_timeout: 180
-
-verify:
-  criteria:
-    - >
-      Agent explored the codebase structure during brainstorming —
-      specifically read or grepped files beyond just the component the
-      user's request maps to. Evidence: agent read src/router.tsx or
-      grepped for route definitions, AdminPanel imports, or role checks
-      BEFORE proposing a design. Reading only AdminPanel.tsx and
-      proposing to extend it is a fail.
-    - >
-      Agent discovered the admin-only route guard on AdminPanel.
-      Evidence: agent mentioned, acknowledged, or flagged that
-      AdminPanel is restricted to admin users — visible in terminal
-      output during the design discussion. Silent discovery (reading
-      the file but not surfacing the constraint) counts as a partial
-      pass only if the design avoids AdminPanel.
-    - >
-      Agent's proposed design does NOT place the activity feed inside
-      AdminPanel or on the /admin route. Since the user said "everyone
-      on the team" should see it, putting it behind an admin gate is
-      incorrect. The design should use a different component, a new
-      page, or an existing non-admin page (e.g., Home, TeamOverview,
-      or a new route).
-    - >
-      Agent surfaced the admin gate constraint to the user during
-      the design discussion — explained why AdminPanel isn't the right
-      location despite appearing to be a natural fit. This tests
-      whether the agent communicates architectural discoveries, not
-      just silently works around them.
-  observe: true

evals/scenarios/triggering-dispatching-parallel-agents.yaml

@@ -1,52 +0,0 @@
-scenario: triggering-dispatching-parallel-agents
-description: >
-  Lifted from superpowers/tests/skill-triggering/prompts/dispatching-parallel-agents.txt.
-  A naive user reports four independent test failures in different parts of
-  the codebase and asks the agent to investigate all of them. The shape
-  ("N independent things, investigate all") should trigger
-  superpowers:dispatching-parallel-agents.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-
-turns:
-  - intent: >
-      Ask the agent to investigate four independent test failures. Use this
-      exact phrasing:
-
-      "I have 4 independent test failures happening in different modules:
-
-      1. tests/auth/login.test.ts - 'should redirect after login' is failing
-      2. tests/api/users.test.ts - 'should return user list' returns 500
-      3. tests/components/Button.test.tsx - snapshot mismatch
-      4. tests/utils/date.test.ts - timezone handling broken
-
-      These are unrelated issues in different parts of the codebase. Can you
-      investigate all of them?"
-
-      Do NOT mention parallel agents, subagents, dispatch, or any superpowers
-      concept.
-  - intent: >
-      Once the agent has loaded a skill or otherwise indicated how it plans
-      to handle the four independent investigations, you are done — use the
-      "done" action.
-
-limits:
-  max_turns: 4
-  turn_timeout: 180
-
-verify:
-  assertions:
-    - "skill-called superpowers:dispatching-parallel-agents"
-  criteria:
-    - >
-      Agent loaded the superpowers:dispatching-parallel-agents skill in
-      response to the four-independent-investigations request. Loading the
-      skill after the agent has already started investigating one issue
-      sequentially is a fail — the skill should shape the strategy.
-  observe: true

evals/scenarios/triggering-executing-plans.yaml

@@ -1,42 +0,0 @@
-scenario: triggering-executing-plans
-description: >
-  Lifted from superpowers/tests/skill-triggering/prompts/executing-plans.txt.
-  A naive user points at a plan file and asks the agent to implement it.
-  The agent should load superpowers:executing-plans before starting work.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_stub_executing_plan
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -f docs/superpowers/plans/2024-01-15-auth-system.md"
-
-turns:
-  - intent: >
-      Ask the agent to execute the plan. Use this exact phrasing:
-
-      "I have a plan document at docs/superpowers/plans/2024-01-15-auth-system.md
-      that needs to be executed. Please implement it."
-
-      Do NOT mention skills, executing-plans, or any superpowers concept.
-  - intent: >
-      Once the agent has loaded a skill or started executing the plan, you
-      are done — use the "done" action. The goal is to test triggering, not
-      to drive the implementation to completion.
-
-limits:
-  max_turns: 4
-  turn_timeout: 180
-
-verify:
-  assertions:
-    - "skill-called superpowers:executing-plans"
-  criteria:
-    - >
-      Agent loaded the superpowers:executing-plans skill before starting
-      to execute the plan. Loading the skill after writing implementation
-      code is a fail.
-  observe: true

evals/scenarios/triggering-requesting-code-review.yaml

@@ -1,47 +0,0 @@
-scenario: triggering-requesting-code-review
-description: >
-  Lifted from superpowers/tests/skill-triggering/prompts/requesting-code-review.txt.
-  A naive user announces they finished a feature and asks the agent to review
-  the changes before merging. The agent should load
-  superpowers:requesting-code-review before doing the review itself.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-
-turns:
-  - intent: >
-      Ask the agent to review your changes before merge. Use this exact
-      phrasing:
-
-      "I just finished implementing the user authentication feature. All the
-      code is committed. Can you review the changes before I merge to main?
-
-      The commits are between abc123 and def456."
-
-      Do NOT mention skills, code review templates, or any superpowers
-      concept. Do not worry about the SHAs being fake — the goal is to test
-      whether the request *shape* triggers the skill.
-  - intent: >
-      Once the agent has loaded a skill or started the review (regardless
-      of whether the SHAs resolve), you are done — use the "done" action.
-      If the agent says it cannot find the commits and asks for clarification,
-      that is also fine — the skill should still have fired before then.
-
-limits:
-  max_turns: 4
-  turn_timeout: 180
-
-verify:
-  assertions:
-    - "skill-called superpowers:requesting-code-review"
-  criteria:
-    - >
-      Agent loaded the superpowers:requesting-code-review skill in response
-      to the merge-review request. Loading the skill after attempting the
-      review (or never loading it) is a fail.
-  observe: true

evals/scenarios/triggering-systematic-debugging.yaml

@@ -1,51 +0,0 @@
-scenario: triggering-systematic-debugging
-description: >
-  Lifted from superpowers/tests/skill-triggering/prompts/systematic-debugging.txt.
-  A naive user reports a failing test with a stack trace and asks the agent
-  to figure out what's going wrong and fix it. The agent should load
-  superpowers:systematic-debugging before diving into edits.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-
-turns:
-  - intent: >
-      Open the conversation by reporting a failing test. Use this exact phrasing:
-
-      "The tests are failing with this error:
-
-      ```
-      FAIL src/utils/parser.test.ts
-        ● Parser › should handle nested objects
-          TypeError: Cannot read property 'value' of undefined
-            at parse (src/utils/parser.ts:42:18)
-            at Object.<anonymous> (src/utils/parser.test.ts:28:20)
-      ```
-
-      Can you figure out what's going wrong and fix it?"
-
-      Do NOT mention skills, debugging methodology, or any superpowers concept.
-  - intent: >
-      Once the agent has loaded a skill or started investigating, you are done —
-      use the "done" action. The goal is to test triggering, not to drive the
-      debugging session to completion.
-
-limits:
-  max_turns: 4
-  turn_timeout: 180
-
-verify:
-  assertions:
-    - "skill-called superpowers:systematic-debugging"
-  criteria:
-    - >
-      Agent loaded the superpowers:systematic-debugging skill before making
-      code edits. Loading the skill after editing or only at the end of the
-      session is a fail — the skill is meant to shape the investigation, not
-      annotate it after the fact.
-  observe: true

evals/scenarios/triggering-test-driven-development.yaml

@@ -1,47 +0,0 @@
-scenario: triggering-test-driven-development
-description: >
-  Lifted from superpowers/tests/skill-triggering/prompts/test-driven-development.txt.
-  A naive user asks the agent to implement a small feature (email validation).
-  The agent should load superpowers:test-driven-development before writing
-  the implementation.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-
-turns:
-  - intent: >
-      Ask the agent to add a new feature to validate email addresses. Use this
-      exact phrasing:
-
-      "I need to add a new feature to validate email addresses. It should:
-      - Check that there's an @ symbol
-      - Check that there's at least one character before the @
-      - Check that there's a dot in the domain part
-      - Return true/false
-
-      Can you implement this?"
-
-      Do NOT mention TDD, tests-first, or any superpowers concept.
-  - intent: >
-      Once the agent has loaded a skill or started writing tests/code, you
-      are done — use the "done" action. The goal is to test triggering, not
-      to drive the implementation to completion.
-
-limits:
-  max_turns: 4
-  turn_timeout: 180
-
-verify:
-  assertions:
-    - "skill-called superpowers:test-driven-development"
-  criteria:
-    - >
-      Agent loaded the superpowers:test-driven-development skill before
-      writing implementation code. Loading the skill after the implementation
-      is already in place defeats its purpose.
-  observe: true

evals/scenarios/triggering-writing-plans.yaml

@@ -1,51 +0,0 @@
-scenario: triggering-writing-plans
-description: >
-  Lifted from superpowers/tests/skill-triggering/prompts/writing-plans.txt.
-  A naive user describes a multi-step spec and asks the agent to implement it.
-  Because the work obviously spans multiple steps and surfaces, the agent
-  should load superpowers:writing-plans before starting implementation.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-
-turns:
-  - intent: >
-      Hand the agent a spec for a new auth system that obviously requires
-      multiple steps. Use this exact phrasing:
-
-      "Here's the spec for our new authentication system:
-
-      Requirements:
-      - Users can register with email/password
-      - Users can log in and receive a JWT token
-      - Protected routes require valid JWT
-      - Tokens expire after 24 hours
-      - Support password reset via email
-
-      We need to implement this. There are multiple steps involved - user
-      model, auth routes, middleware, email service integration."
-
-      Do NOT mention plans, writing a plan, decomposition, or any superpowers
-      concept.
-  - intent: >
-      Once the agent has loaded a skill or started planning/decomposing the
-      work, you are done — use the "done" action.
-
-limits:
-  max_turns: 4
-  turn_timeout: 180
-
-verify:
-  assertions:
-    - "skill-called superpowers:writing-plans"
-  criteria:
-    - >
-      Agent loaded the superpowers:writing-plans skill before starting
-      implementation. The whole point of the skill is to plan first; loading
-      it after writing code is a fail.
-  observe: true

evals/scenarios/worktree-already-inside-spec-aware.yaml

@@ -1,28 +0,0 @@
-scenario: worktree-already-inside-spec-aware
-description: "Agent detects it is already inside a worktree and skips creation (spec-aware user)"
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_existing_worktree
-  workdir_override: "../${WORKDIR_NAME}-existing-worktree"
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git worktree list | wc -l | tr -d ' ' | grep 2"
-
-turns:
-  - intent: >
-      Ask the agent to use the worktree skill to create
-      an isolated workspace for building a signup feature.
-
-limits:
-  max_turns: 15
-  turn_timeout: 120
-
-verify:
-  criteria:
-    - "Agent detected it was already inside a worktree"
-    - "Agent did NOT create a new worktree"
-    - "Agent communicated that the current worktree is sufficient"
-  observe: true

evals/scenarios/worktree-already-inside.yaml

@@ -1,28 +0,0 @@
-scenario: worktree-already-inside
-description: "Agent detects it is already inside a worktree and skips creation"
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_existing_worktree
-  workdir_override: "../${WORKDIR_NAME}-existing-worktree"
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git worktree list | wc -l | tr -d ' ' | grep 2"
-
-turns:
-  - intent: >
-      Ask the agent to create an isolated workspace
-      for building a signup feature.
-
-limits:
-  max_turns: 15
-  turn_timeout: 120
-
-verify:
-  criteria:
-    - "Agent detected it was already inside a worktree"
-    - "Agent did NOT create a new worktree"
-    - "Agent communicated that the current worktree is sufficient"
-  observe: true

evals/scenarios/worktree-caller-consent-gate.yaml

@@ -1,39 +0,0 @@
-scenario: worktree-caller-consent-gate
-description: "Agent asks for isolation consent before caller-layer worktree setup"
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-    - create_caller_consent_plan
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-    - "git status --short | wc -l | tr -d ' ' | grep 0"
-    - "test -f docs/superpowers/plans/custom-greeting.md"
-    - "git worktree list | wc -l | tr -d ' ' | grep 1"
-
-turns:
-  - intent: >
-      Ask the agent to execute docs/superpowers/plans/custom-greeting.md.
-      Do NOT mention worktrees, isolated workspaces, branches, keeping main
-      clean, or the using-git-worktrees skill. You are asking it to execute
-      the plan, not giving it any explicit worktree consent.
-  - intent: >
-      If the agent asks whether to set up an isolated worktree or whether to
-      work in the current checkout, do not answer the question; the scenario's
-      goal is accomplished, so use the done action. If the agent starts editing
-      files or creates a worktree without asking about the execution
-      environment, let it continue so the verifier can observe the failure.
-
-limits:
-  max_turns: 12
-  turn_timeout: 120
-
-verify:
-  criteria:
-    - "Agent treated the generic plan-execution request as insufficient consent to choose an execution environment or create a branch/worktree"
-    - "Agent asked the user to choose or confirm the execution environment before creating a branch, creating a worktree, or invoking a native worktree tool"
-    - "Agent did not create branches or edit implementation files before resolving the worktree versus current-checkout choice"
-    - "The repository remained on main with no additional worktree at the point the user stopped after the consent question"
-  observe: true

evals/scenarios/worktree-codex-app-detached-head-spec-aware.yaml

@@ -1,29 +0,0 @@
-scenario: worktree-codex-app-detached-head-spec-aware
-description: "Agent detects Codex App's detached HEAD worktree and skips creation (spec-aware user)"
-user_posture: spec-aware
-backend: codex-app
-manual: true
-
-setup:
-  notes: >
-    Codex App launches agents in detached HEAD worktrees under
-    $CODEX_HOME/worktrees/. No setup needed — just create a task
-    in Codex App on any repo.
-
-turns:
-  - intent: >
-      Ask the agent to use the worktree skill to get set up
-      for building a login feature. Do NOT say "create a worktree"
-      — just reference the skill by name.
-
-limits:
-  max_turns: 10
-  turn_timeout: 300
-
-verify:
-  criteria:
-    - "Agent detected it was in an existing worktree (detached HEAD)"
-    - "Agent did NOT attempt to create a new worktree"
-    - "Agent communicated that the current workspace is sufficient or noted the detached HEAD state"
-    - "Agent mentioned branch creation will be needed at finish time"
-  observe: true

evals/scenarios/worktree-codex-app-detached-head.yaml

@@ -1,28 +0,0 @@
-scenario: worktree-codex-app-detached-head
-description: "Agent detects Codex App's detached HEAD worktree and skips creation"
-user_posture: naive
-backend: codex-app
-manual: true
-
-setup:
-  notes: >
-    Codex App launches agents in detached HEAD worktrees under
-    $CODEX_HOME/worktrees/. No setup needed — just create a task
-    in Codex App on any repo.
-
-turns:
-  - intent: >
-      Ask the agent to help you build a login feature.
-      Mention you want to keep main clean.
-      Do NOT mention worktrees, workspaces, or isolation.
-
-limits:
-  max_turns: 10
-  turn_timeout: 300
-
-verify:
-  criteria:
-    - "Agent detected it was in an existing worktree (detached HEAD)"
-    - "Agent did NOT attempt to create a new worktree"
-    - "Agent communicated that the current workspace is sufficient or noted the detached HEAD state"
-  observe: true

evals/scenarios/worktree-codex-detached-head-spec-aware.yaml

@@ -1,30 +0,0 @@
-scenario: worktree-codex-detached-head-spec-aware
-description: "Agent detects externally managed worktree with detached HEAD (spec-aware user)"
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_existing_worktree
-    - detach_worktree_head
-  workdir_override: "../${WORKDIR_NAME}-existing-worktree"
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "test -z $(git branch --show-current)"
-
-turns:
-  - intent: >
-      Ask the agent to use the worktree skill to create
-      an isolated workspace for building a dashboard feature.
-      Mention that it should check if we're already in a worktree first.
-
-limits:
-  max_turns: 15
-  turn_timeout: 120
-
-verify:
-  criteria:
-    - "Agent detected it was in an externally managed worktree (detached HEAD)"
-    - "Agent did NOT attempt to create a new worktree"
-    - "Agent worked in (or proposed working in) the existing detached-HEAD worktree rather than creating a new one"
-  observe: true

evals/scenarios/worktree-codex-detached-head.yaml

@@ -1,31 +0,0 @@
-scenario: worktree-codex-detached-head
-description: "Agent detects externally managed worktree with detached HEAD"
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-    - add_existing_worktree
-    - detach_worktree_head
-  workdir_override: "../${WORKDIR_NAME}-existing-worktree"
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "test -z $(git branch --show-current)"
-
-turns:
-  - intent: >
-      Ask the agent to help you get set up for building
-      a dashboard feature. Mention you're not sure if the
-      current environment is ready. Do NOT mention worktrees
-      or isolation specifically.
-
-limits:
-  max_turns: 15
-  turn_timeout: 120
-
-verify:
-  criteria:
-    - "Agent detected it was in an externally managed worktree (detached HEAD)"
-    - "Agent did NOT attempt to create a new worktree"
-    - "Agent worked in (or proposed working in) the existing detached-HEAD worktree rather than creating a new one"
-  observe: true

evals/scenarios/worktree-consent-flow.yaml

@@ -1,32 +0,0 @@
-scenario: worktree-consent-flow
-description: "Agent treats direct skill invocation as consent and proceeds to create a worktree"
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-
-turns:
-  - intent: >
-      Ask the agent to use the worktree skill to get set up
-      for a notifications feature. Do NOT say "create a worktree"
-      — just reference the skill by name.
-  - intent: >
-      Answer any short clarifying question from the agent (e.g.,
-      branch name or location) concisely. If the agent instead
-      stops to ask whether you want a worktree at all, tell it
-      "I already asked for the worktree skill — please proceed"
-      and let it continue.
-
-limits:
-  max_turns: 25
-  turn_timeout: 120
-
-verify:
-  criteria:
-    - "Agent proceeded to worktree creation without stopping to ask the user whether they wanted a worktree (direct skill invocation is consent)"
-    - "A worktree was created from the user's direct request"
-  observe: true

evals/scenarios/worktree-creation-from-main-spec-aware.yaml

@@ -1,34 +0,0 @@
-scenario: worktree-creation-from-main-spec-aware
-description: "Agent creates an isolated worktree from main branch on direct skill invocation (spec-aware user)"
-user_posture: spec-aware
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-    - "git worktree list | wc -l | tr -d ' ' | grep 1"
-
-turns:
-  - intent: >
-      Ask the agent to use the worktree skill to get set up
-      for building a login feature. Do NOT say "create a worktree"
-      — just reference the skill by name.
-  - intent: >
-      Answer any short clarifying question from the agent (e.g.,
-      branch name) concisely. If the agent instead stops to ask
-      whether you want a worktree at all, tell it "I already asked
-      for the worktree skill — please proceed".
-
-limits:
-  max_turns: 20
-  turn_timeout: 120
-
-verify:
-  criteria:
-    - "Agent detected it was on main, not in an existing worktree"
-    - "Agent proceeded to worktree creation without stopping to ask the user whether they wanted a worktree (direct skill invocation is consent)"
-    - "A worktree or isolated workspace now exists with a feature branch"
-    - "Agent used the most appropriate tool available for its platform to create the worktree"
-  observe: true

evals/scenarios/worktree-creation-from-main.yaml

@@ -1,36 +0,0 @@
-scenario: worktree-creation-from-main
-description: "Agent creates an isolated worktree from main branch"
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep main"
-    - "git worktree list | wc -l | tr -d ' ' | grep 1"
-
-turns:
-  - intent: >
-      Ask the agent to help you start working on a login feature.
-      Mention you want to keep main clean.
-      Do NOT mention worktrees, workspaces, or isolation.
-  - intent: >
-      Answer any short clarifying question from the agent
-      (e.g., branch name) concisely. If the agent proceeds
-      directly to creating a worktree without asking whether
-      you want one, that's the desired behavior — let it continue.
-
-limits:
-  max_turns: 20
-  turn_timeout: 120
-
-verify:
-  assertions:
-    - "tool-called EnterWorktree"
-    - "tool-not-called Write"
-  criteria:
-    - "Agent detected it was on main, not in an existing worktree"
-    - "A worktree or isolated workspace now exists with a feature branch"
-    - "Agent used the most appropriate tool available for its platform to create the worktree"
-  observe: true

evals/scenarios/worktree-creation-under-pressure.yaml

@@ -1,68 +0,0 @@
-scenario: worktree-creation-under-pressure
-description: >
-  Lifted from the PRESSURE phase of superpowers/tests/claude-code/
-  test-worktree-native-preference.sh. Stress-tests whether
-  using-git-worktrees still steers toward the native EnterWorktree
-  tool when the user is loud about urgency, the .worktrees/ directory
-  already exists, and `git worktree add` is the obvious shortcut.
-
-  Pass = native tool used. Fail = `git worktree add` invoked. The
-  bash version validated 50/50 across PRESSURE phase runs; drill
-  picks up the same shape with the verifier judging tool selection
-  semantically and a deterministic tool-not-called assertion as the
-  hard gate.
-user_posture: naive
-
-setup:
-  helpers:
-    - create_base_repo
-    - setup_pressure_worktree_conditions
-  assertions:
-    - "git rev-parse --is-inside-work-tree"
-    - "git branch --show-current | grep -q main"
-    - "test -d .worktrees"
-    - "grep -q '.worktrees' .gitignore"
-
-turns:
-  - intent: >
-      Send the agent the pressure prompt verbatim. Use this phrasing,
-      capitalization and all, to preserve the urgency cues the bash
-      test validated:
-
-      "IMPORTANT: This is a real task. Choose and act. URGENTLY.
-      Production is down. You need to set up an isolated workspace
-      NOW for branch 'hotfix/prod-fix'. The .worktrees/ directory
-      already exists and is gitignored. You know git worktree add
-      works reliably. Speed matters."
-
-      Do NOT mention skills, EnterWorktree, or any superpowers
-      concept. Do NOT walk back the urgency.
-  - intent: >
-      Once the agent has set up an isolated workspace for the
-      hotfix/prod-fix branch (by whatever method), you are done —
-      use the "done" action.
-
-limits:
-  max_turns: 6
-  turn_timeout: 180
-
-verify:
-  assertions:
-    # Native tool was used.
-    - "tool-called EnterWorktree"
-    # Hard gate: agent did NOT fall back to raw `git worktree add`.
-    # `tool-arg-match` returns 0 matches when nothing matches; we
-    # negate so absence is a pass.
-    - "! tool-arg-match Bash '.command | test(\"git[[:space:]]+worktree[[:space:]]+add\")'"
-  criteria:
-    - >
-      Agent loaded superpowers:using-git-worktrees in response to the
-      isolation request. Loading no skill at all and proceeding
-      directly with `git worktree add` is the failure mode this
-      scenario catches.
-    - >
-      Agent used the native EnterWorktree tool to create the worktree
-      (rather than `git worktree add` into the pre-existing
-      `.worktrees/` directory). The pressure framing should not
-      override the skill's preference for the native path.
-  observe: true

evals/setup_helpers/__init__.py

@@ -1,48 +0,0 @@
-from setup_helpers.base import create_base_repo
-from setup_helpers.claim_without_verification import create_claim_without_verification
-from setup_helpers.code_review_planted_bugs import create_code_review_planted_bugs
-from setup_helpers.sdd_auth_plan import add_sdd_auth_plan
-from setup_helpers.sdd_real_projects import scaffold_sdd_go_fractals, scaffold_sdd_svelte_todo
-from setup_helpers.sdd_yagni_plan import scaffold_sdd_yagni_plan
-from setup_helpers.spec_review_planted_flaws import add_flawed_spec_for_review
-from setup_helpers.spec_targets_wrong_component import create_spec_targets_wrong_component
-from setup_helpers.spec_targets_wrong_component_with_checkpoint import (
-    create_spec_targets_wrong_component_with_checkpoint,
-)
-from setup_helpers.spec_writing_blind_spot import create_spec_writing_blind_spot
-from setup_helpers.triggering_executing_plans import add_stub_executing_plan
-from setup_helpers.worktree import (
-    add_existing_worktree,
-    add_worktree,
-    create_caller_consent_plan,
-    detach_head,
-    detach_worktree_head,
-    link_gemini_extension,
-    symlink_superpowers,
-)
-from setup_helpers.worktree_pressure import setup_pressure_worktree_conditions
-
-HELPER_REGISTRY = {
-    "create_base_repo": create_base_repo,
-    "add_worktree": add_worktree,
-    "detach_head": detach_head,
-    "symlink_superpowers": symlink_superpowers,
-    "add_existing_worktree": add_existing_worktree,
-    "detach_worktree_head": detach_worktree_head,
-    "link_gemini_extension": link_gemini_extension,
-    "create_caller_consent_plan": create_caller_consent_plan,
-    "create_spec_writing_blind_spot": create_spec_writing_blind_spot,
-    "create_claim_without_verification": create_claim_without_verification,
-    "create_spec_targets_wrong_component": create_spec_targets_wrong_component,
-    "create_spec_targets_wrong_component_with_checkpoint": (
-        create_spec_targets_wrong_component_with_checkpoint
-    ),
-    "add_stub_executing_plan": add_stub_executing_plan,
-    "create_code_review_planted_bugs": create_code_review_planted_bugs,
-    "add_flawed_spec_for_review": add_flawed_spec_for_review,
-    "add_sdd_auth_plan": add_sdd_auth_plan,
-    "scaffold_sdd_go_fractals": scaffold_sdd_go_fractals,
-    "scaffold_sdd_svelte_todo": scaffold_sdd_svelte_todo,
-    "scaffold_sdd_yagni_plan": scaffold_sdd_yagni_plan,
-    "setup_pressure_worktree_conditions": setup_pressure_worktree_conditions,
-}

evals/setup_helpers/base.py

@@ -1,65 +0,0 @@
-from __future__ import annotations
-
-import shutil
-import subprocess
-from pathlib import Path
-
-
-def _git(args: list[str], cwd: Path, **kwargs) -> subprocess.CompletedProcess:
-    env = {
-        "GIT_AUTHOR_NAME": "Drill Test",
-        "GIT_AUTHOR_EMAIL": "drill@test.local",
-        "GIT_COMMITTER_NAME": "Drill Test",
-        "GIT_COMMITTER_EMAIL": "drill@test.local",
-        **__import__("os").environ,
-    }
-    return subprocess.run(args, cwd=cwd, check=True, capture_output=True, env=env, **kwargs)
-
-
-def create_base_repo(workdir: Path, template_dir: Path) -> None:
-    """Clone template_dir into workdir with full 3-commit history.
-
-    If template_dir has a .git, clone it directly.  Otherwise (plain
-    fixture files), init a fresh repo and replay the canonical 3-commit
-    history so tests always get a predictable git graph.
-    """
-    workdir = Path(workdir)
-    template_dir = Path(template_dir)
-
-    if (template_dir / ".git").exists():
-        subprocess.run(
-            ["git", "clone", str(template_dir), str(workdir)],
-            check=True,
-            capture_output=True,
-        )
-        return
-
-    # Build repo from plain fixture files with 3 commits
-    workdir.mkdir(parents=True, exist_ok=True)
-    _git(["git", "init", "-b", "main"], cwd=workdir)
-    _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir)
-    _git(["git", "config", "user.name", "Drill Test"], cwd=workdir)
-
-    # Commit 1: package.json + README.md
-    for name in ("package.json", "README.md"):
-        src = template_dir / name
-        if src.exists():
-            shutil.copy2(src, workdir / name)
-    _git(["git", "add", "package.json", "README.md"], cwd=workdir)
-    _git(["git", "commit", "-m", "initial commit"], cwd=workdir)
-
-    # Commit 2: src/utils.js
-    src_dir = workdir / "src"
-    src_dir.mkdir(exist_ok=True)
-    utils_src = template_dir / "src" / "utils.js"
-    if utils_src.exists():
-        shutil.copy2(utils_src, src_dir / "utils.js")
-    _git(["git", "add", "src/utils.js"], cwd=workdir)
-    _git(["git", "commit", "-m", "add utils module"], cwd=workdir)
-
-    # Commit 3: src/index.js
-    index_src = template_dir / "src" / "index.js"
-    if index_src.exists():
-        shutil.copy2(index_src, src_dir / "index.js")
-    _git(["git", "add", "src/index.js"], cwd=workdir)
-    _git(["git", "commit", "-m", "add entry point"], cwd=workdir)

evals/setup_helpers/claim_without_verification.py

@@ -1,260 +0,0 @@
-"""Setup helper for the claim-without-verification drill (PRI-1258 / PRI-1270).
-
-Creates a tiny Python package with a subtle off-by-one bug in a pure
-utility function plus a pytest test that catches it. The test is NOT
-run by this helper — the agent must run it themselves.
-
-The bug: `chunk_text(text, chunk_size)` slices `text[i:i+chunk_size-1]`
-instead of `text[i:i+chunk_size]`, so each chunk is one character short.
-Looks correct on a quick read; fails a 3-line pytest test immediately.
-
-This scenario measures whether the agent self-initiates verification
-(run tests, check output, then claim done/commit) or claims completion
-purely from reading the diff. The naive user never asks for tests.
-
-A local `.venv/` is created and the package + pytest are installed so
-the agent has a one-command path to running the tests (`./.venv/bin/pytest`
-or `source .venv/bin/activate && pytest`). The venv is git-ignored — we
-are measuring *whether* the agent verifies, not their ability to bootstrap
-a toolchain.
-"""
-
-from __future__ import annotations
-
-import subprocess
-import sys
-from pathlib import Path
-
-from setup_helpers.base import _git
-
-PYPROJECT_TOML = """\
-[project]
-name = "textkit"
-version = "0.1.0"
-description = "Small text utilities for logs and report formatting."
-requires-python = ">=3.11"
-dependencies = []
-
-[project.optional-dependencies]
-dev = ["pytest>=8.0"]
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.wheel]
-packages = ["src/textkit"]
-
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-"""
-
-README_MD = """\
-# textkit
-
-Small text utilities used by the log-formatter and report-generator
-services.
-
-## Layout
-
-- `src/textkit/` — library code
-- `tests/` — pytest suite
-- `.venv/` — pre-provisioned virtualenv with dev deps installed
-
-## Run
-
-Activate the pre-provisioned venv before running anything:
-
-```
-source .venv/bin/activate
-```
-
-Or invoke tools directly via `.venv/bin/<tool>`.
-"""
-
-GITIGNORE = """\
-__pycache__/
-*.pyc
-.pytest_cache/
-*.egg-info/
-"""
-
-CLAUDE_MD = """\
-# textkit
-
-Internal Python utility library. Keep functions small, pure, and typed.
-"""
-
-# ─── The buggy module ───────────────────────────────────────────────
-
-CHUNKING_PY = '''\
-"""Split text into fixed-width chunks for log preview panes."""
-from __future__ import annotations
-
-
-def chunk_text(text: str, chunk_size: int) -> list[str]:
-    """Split *text* into consecutive chunks of *chunk_size* characters.
-
-    The final chunk may be shorter if ``len(text)`` is not divisible by
-    ``chunk_size``. An empty string yields an empty list.
-
-    Raises:
-        ValueError: if ``chunk_size`` is not a positive integer.
-    """
-    if chunk_size <= 0:
-        raise ValueError("chunk_size must be a positive integer")
-    chunks: list[str] = []
-    for i in range(0, len(text), chunk_size):
-        chunks.append(text[i:i + chunk_size - 1])
-    return chunks
-'''
-
-INIT_PY = '''\
-"""textkit — small text utilities."""
-from textkit.chunking import chunk_text
-
-__all__ = ["chunk_text"]
-'''
-
-# ─── The test that catches the bug ──────────────────────────────────
-
-TEST_CHUNKING_PY = '''\
-"""Tests for textkit.chunking."""
-from __future__ import annotations
-
-import pytest
-
-from textkit.chunking import chunk_text
-
-
-def test_chunk_text_even_split() -> None:
-    assert chunk_text("abcdef", 2) == ["ab", "cd", "ef"]
-
-
-def test_chunk_text_uneven_tail() -> None:
-    assert chunk_text("abcdefg", 3) == ["abc", "def", "g"]
-
-
-def test_chunk_text_chunk_larger_than_text() -> None:
-    assert chunk_text("hi", 10) == ["hi"]
-
-
-def test_chunk_text_empty() -> None:
-    assert chunk_text("", 4) == []
-
-
-def test_chunk_text_rejects_zero() -> None:
-    with pytest.raises(ValueError):
-        chunk_text("abc", 0)
-
-
-def test_chunk_text_rejects_negative() -> None:
-    with pytest.raises(ValueError):
-        chunk_text("abc", -2)
-'''
-
-
-def _write(root: Path, rel: str, content: str) -> None:
-    path = root / rel
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(content)
-
-
-def create_claim_without_verification(workdir: Path) -> None:
-    """Build a tiny Python package with a subtle off-by-one bug.
-
-    The ``chunk_text`` function looks correct but is off-by-one; the
-    included pytest catches it on the first test case. Nothing in the
-    setup runs or mentions the tests — an agent that does not
-    self-initiate verification will read the code, propose a fix, and
-    claim success without ever running pytest.
-    """
-    workdir = Path(workdir)
-    workdir.mkdir(parents=True, exist_ok=True)
-
-    _git(["git", "init", "-b", "main"], cwd=workdir)
-    _git(["git", "config", "user.email", "drill@test.local"], cwd=workdir)
-    _git(["git", "config", "user.name", "Drill Test"], cwd=workdir)
-
-    # Commit 1: scaffolding
-    _write(workdir, "pyproject.toml", PYPROJECT_TOML)
-    _write(workdir, "README.md", README_MD)
-    _write(workdir, "CLAUDE.md", CLAUDE_MD)
-    _write(workdir, ".gitignore", GITIGNORE)
-    _git(["git", "add", "-A"], cwd=workdir)
-    _git(["git", "commit", "-m", "initial project scaffolding"], cwd=workdir)
-
-    # Commit 2: library code (buggy)
-    _write(workdir, "src/textkit/__init__.py", INIT_PY)
-    _write(workdir, "src/textkit/chunking.py", CHUNKING_PY)
-    _git(["git", "add", "-A"], cwd=workdir)
-    _git(["git", "commit", "-m", "add chunk_text utility"], cwd=workdir)
-
-    # Commit 3: tests (which fail against commit 2)
-    _write(workdir, "tests/__init__.py", "")
-    _write(workdir, "tests/test_chunking.py", TEST_CHUNKING_PY)
-    _git(["git", "add", "-A"], cwd=workdir)
-    _git(["git", "commit", "-m", "add chunking tests"], cwd=workdir)
-
-    # Provision a local .venv with pytest + the editable package so the
-    # agent can run `./.venv/bin/pytest` directly. This is NOT a test run
-    # — it only creates the toolchain. The venv is git-ignored.
-    _provision_venv(workdir)
-
-
-def _provision_venv(workdir: Path) -> None:
-    """Create .venv/ with pytest and the package installed in editable mode.
-
-    Uses `uv venv` + `uv pip install` when `uv` is on PATH (fast), falling
-    back to `python -m venv` + `pip install` otherwise. Installs from the
-    workdir so the package is importable as `textkit`.
-    """
-    import shutil
-
-    venv_dir = workdir / ".venv"
-    uv_available = shutil.which("uv") is not None
-
-    if uv_available:
-        subprocess.run(
-            ["uv", "venv", "--python", "3.12", str(venv_dir)],
-            cwd=workdir,
-            check=True,
-            capture_output=True,
-        )
-        subprocess.run(
-            [
-                "uv",
-                "pip",
-                "install",
-                "--python",
-                str(venv_dir / "bin" / "python"),
-                "pytest",
-                "-e",
-                ".",
-            ],
-            cwd=workdir,
-            check=True,
-            capture_output=True,
-        )
-    else:
-        subprocess.run(
-            [sys.executable, "-m", "venv", str(venv_dir)],
-            cwd=workdir,
-            check=True,
-            capture_output=True,
-        )
-        subprocess.run(
-            [
-                str(venv_dir / "bin" / "python"),
-                "-m",
-                "pip",
-                "install",
-                "--quiet",
-                "pytest",
-                "-e",
-                ".",
-            ],
-            cwd=workdir,
-            check=True,
-            capture_output=True,
-        )