mirror of
https://github.com/affaan-m/ECC.git
synced 2026-09-08 07:37:48 +02:00
fix(gan): grant evaluator Playwright tools
This commit is contained in:
committed by
Alex Schmitt
parent
b7faf3d70e
commit
ef68f816d1
+15
-1
@@ -1,7 +1,7 @@
|
|||||||
---
|
---
|
||||||
name: gan-evaluator
|
name: gan-evaluator
|
||||||
description: "GAN Harness — Evaluator agent. Tests the live running application via Playwright, scores against rubric, and provides actionable feedback to the Generator."
|
description: "GAN Harness — Evaluator agent. Tests the live running application via Playwright, scores against rubric, and provides actionable feedback to the Generator."
|
||||||
tools: Read, Write, Bash, Grep, Glob
|
tools: Read, Write, Bash, Grep, Glob, mcp__playwright__browser_navigate, mcp__playwright__browser_click, mcp__playwright__browser_take_screenshot, mcp__playwright__browser_snapshot, mcp__playwright__browser_type, mcp__playwright__browser_fill_form
|
||||||
model: sonnet
|
model: sonnet
|
||||||
color: red
|
color: red
|
||||||
---
|
---
|
||||||
@@ -35,6 +35,12 @@ You are the QA Engineer and Design Critic. You test the **live running applicati
|
|||||||
|
|
||||||
## Evaluation Workflow
|
## Evaluation Workflow
|
||||||
|
|
||||||
|
Before testing, record the mode that is actually available. The requested mode
|
||||||
|
is not proof that its tools were available: if the Playwright MCP tools cannot
|
||||||
|
be called, switch to the documented `screenshot` or `code-only` fallback and
|
||||||
|
report that degradation instead of silently scoring a static review as a live
|
||||||
|
browser evaluation.
|
||||||
|
|
||||||
### Step 1: Read the Rubric
|
### Step 1: Read the Rubric
|
||||||
```
|
```
|
||||||
Read gan-harness/eval-rubric.md for project-specific criteria
|
Read gan-harness/eval-rubric.md for project-specific criteria
|
||||||
@@ -129,6 +135,14 @@ Write feedback to `gan-harness/feedback/feedback-NNN.md`:
|
|||||||
|
|
||||||
## Scores
|
## Scores
|
||||||
|
|
||||||
|
## Evaluation Mode
|
||||||
|
|
||||||
|
**Achieved:** `playwright` | `screenshot` | `code-only`
|
||||||
|
|
||||||
|
State the mode that was actually completed (not merely the mode requested by
|
||||||
|
the harness). If the requested mode was unavailable, briefly explain why and
|
||||||
|
which fallback was used.
|
||||||
|
|
||||||
| Criterion | Score | Weight | Weighted |
|
| Criterion | Score | Weight | Weighted |
|
||||||
|-----------|-------|--------|----------|
|
|-----------|-------|--------|----------|
|
||||||
| Design Quality | X/10 | 0.3 | X.X |
|
| Design Quality | X/10 | 0.3 | X.X |
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
/**
|
||||||
|
* Regression coverage for the GAN evaluator's live-browser capability.
|
||||||
|
*
|
||||||
|
* Run with: node tests/ci/gan-evaluator-tools.test.js
|
||||||
|
*/
|
||||||
|
|
||||||
|
const assert = require('assert');
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
|
||||||
|
const evaluatorPath = path.join(__dirname, '..', '..', 'agents', 'gan-evaluator.md');
|
||||||
|
const content = fs.readFileSync(evaluatorPath, 'utf8');
|
||||||
|
const frontmatter = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
||||||
|
|
||||||
|
assert.ok(frontmatter, 'gan-evaluator.md should have frontmatter');
|
||||||
|
const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m);
|
||||||
|
assert.ok(toolsLine, 'gan-evaluator.md should declare tools');
|
||||||
|
|
||||||
|
const tools = new Set(toolsLine[1].split(',').map(tool => tool.trim()));
|
||||||
|
for (const tool of [
|
||||||
|
'mcp__playwright__browser_navigate',
|
||||||
|
'mcp__playwright__browser_click',
|
||||||
|
'mcp__playwright__browser_take_screenshot',
|
||||||
|
'mcp__playwright__browser_snapshot',
|
||||||
|
'mcp__playwright__browser_type',
|
||||||
|
'mcp__playwright__browser_fill_form',
|
||||||
|
]) {
|
||||||
|
assert.ok(tools.has(tool), `gan-evaluator.md should grant ${tool}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.match(content, /\*\*Achieved:\*\* `playwright` \| `screenshot` \| `code-only`/);
|
||||||
|
assert.match(content, /mode that was actually completed/);
|
||||||
|
|
||||||
|
console.log('GAN evaluator tools and achieved-mode contract are present.');
|
||||||
Reference in New Issue
Block a user