From 9ff6018675881a373f6acce557e6f1317a9e5193 Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Sun, 26 Jul 2026 05:03:20 -0400 Subject: [PATCH] test: isolate agent tool validator coverage Move scalar-tool validation into a focused discovered test module, bound canonical fixture paths, and guarantee dashboard fixture cleanup even when assertions fail. --- tests/ci/validate-agents-tools.test.js | 175 +++++++++++++++++++++++++ tests/ci/validators.test.js | 66 ---------- tests/scripts/dashboard-web.test.js | 53 +++++--- 3 files changed, 213 insertions(+), 81 deletions(-) create mode 100644 tests/ci/validate-agents-tools.test.js diff --git a/tests/ci/validate-agents-tools.test.js b/tests/ci/validate-agents-tools.test.js new file mode 100644 index 000000000..d2f276281 --- /dev/null +++ b/tests/ci/validate-agents-tools.test.js @@ -0,0 +1,175 @@ +/** + * Focused tests for validate-agents.js tools frontmatter rules. + * + * Run with: node tests/ci/validate-agents-tools.test.js + */ + +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const { execFileSync } = require('child_process'); + +const validatorsDir = path.join(__dirname, '..', '..', 'scripts', 'ci'); +const repoRoot = path.join(__dirname, '..', '..'); +const canonicalAgentsDir = path.join(repoRoot, 'agents'); + +function test(name, fn) { + try { + fn(); + console.log(` \u2713 ${name}`); + return true; + } catch (err) { + console.log(` \u2717 ${name}`); + console.log(` Error: ${err.message}`); + return false; + } +} + +function createTestDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'validate-agents-tools-test-')); +} + +function cleanupTestDir(testDir) { + fs.rmSync(testDir, { recursive: true, force: true }); +} + +function stripShebang(source) { + let s = source; + if (s.charCodeAt(0) === 0xFEFF) s = s.slice(1); + if (s.startsWith('#!')) { + const nl = s.indexOf('\n'); + s = nl === -1 ? '' : s.slice(nl + 1); + } + return s; +} + +function runSourceViaTempFile(source) { + const tmpFile = path.join(repoRoot, `.tmp-validator-${Date.now()}-${Math.random().toString(36).slice(2)}.js`); + try { + fs.writeFileSync(tmpFile, source, 'utf8'); + const stdout = execFileSync('node', [tmpFile], { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 10000, + cwd: repoRoot, + }); + return { code: 0, stdout, stderr: '' }; + } catch (err) { + return { + code: err.status || 1, + stdout: err.stdout || '', + stderr: err.stderr || '', + }; + } finally { + try { fs.unlinkSync(tmpFile); } catch (_) { /* ignore cleanup errors */ } + } +} + +function runValidatorWithDir(validatorName, dirConstant, overridePath) { + const validatorPath = path.join(validatorsDir, `${validatorName}.js`); + let source = fs.readFileSync(validatorPath, 'utf8'); + source = stripShebang(source); + const dirRegex = new RegExp(`const ${dirConstant} = .*?;`); + source = source.replace(dirRegex, `const ${dirConstant} = ${JSON.stringify(overridePath)};`); + return runSourceViaTempFile(source); +} + +function readCanonicalAgent(file) { + const resolvedPath = path.resolve(canonicalAgentsDir, file); + const agentsRoot = path.resolve(canonicalAgentsDir); + assert.ok( + resolvedPath.startsWith(`${agentsRoot}${path.sep}`), + `${file} should resolve inside the canonical agents directory` + ); + return fs.readFileSync(resolvedPath, 'utf8'); +} + +function runTests() { + console.log('\n=== Testing validate-agents tools frontmatter ===\n'); + + let passed = 0; + let failed = 0; + + if (test('canonical agents declare tools as comma-separated scalars', () => { + const agentFiles = fs.readdirSync(canonicalAgentsDir).filter(file => file.endsWith('.md')); + + for (const file of agentFiles) { + const content = readCanonicalAgent(file); + const frontmatter = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); + assert.ok(frontmatter, `${file} should have frontmatter`); + + const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m); + assert.ok(toolsLine, `${file} should declare a non-empty tools scalar`); + assert.ok( + !toolsLine[1].trim().startsWith('['), + `${file} should use comma-separated scalar tools, not a YAML sequence` + ); + } + })) passed++; else failed++; + + if (test('accepts comma-separated scalar agent tools', () => { + const testDir = createTestDir(); + try { + fs.writeFileSync(path.join(testDir, 'scalar-tools.md'), '---\nmodel: sonnet\ntools: Read, Glob, Grep\n---\n# Agent'); + + const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); + assert.strictEqual(result.code, 0, `Should accept scalar tools, got stderr: ${result.stderr}`); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + if (test('rejects YAML sequence-form agent tools', () => { + const testDir = createTestDir(); + try { + fs.writeFileSync(path.join(testDir, 'sequence-tools.md'), '---\nmodel: sonnet\ntools: [Read, Glob, Grep]\n---\n# Agent'); + + const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); + assert.strictEqual(result.code, 1, 'Should reject sequence-form tools'); + assert.ok( + result.stderr.includes('comma-separated scalar'), + `Should explain the supported tools format, got stderr: ${result.stderr}` + ); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + if (test('rejects block sequence-form agent tools', () => { + const testDir = createTestDir(); + try { + fs.writeFileSync(path.join(testDir, 'block-sequence-tools.md'), '---\nmodel: sonnet\ntools:\n - Read\n - Glob\n - Grep\n---\n# Agent'); + + const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); + assert.strictEqual(result.code, 1, 'Should reject block sequence-form tools'); + assert.ok( + result.stderr.includes('comma-separated scalar'), + `Should explain the supported tools format, got stderr: ${result.stderr}` + ); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + if (test('rejects explicitly tagged YAML sequence-form agent tools', () => { + const testDir = createTestDir(); + try { + fs.writeFileSync(path.join(testDir, 'tagged-sequence-tools.md'), '---\nmodel: sonnet\ntools: !!seq [Read, Glob, Grep]\n---\n# Agent'); + + const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); + assert.strictEqual(result.code, 1, 'Should reject tagged sequence-form tools'); + assert.ok( + result.stderr.includes('comma-separated scalar'), + `Should explain the supported tools format, got stderr: ${result.stderr}` + ); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); + process.exit(failed > 0 ? 1 : 0); +} + +runTests(); diff --git a/tests/ci/validators.test.js b/tests/ci/validators.test.js index 5dc825a12..8c34d0029 100644 --- a/tests/ci/validators.test.js +++ b/tests/ci/validators.test.js @@ -334,24 +334,6 @@ function runTests() { assert.ok(result.stdout.includes('Validated'), 'Should output validation count'); })) passed++; else failed++; - if (test('canonical agents declare tools as comma-separated scalars', () => { - const agentsDir = path.join(__dirname, '..', '..', 'agents'); - const agentFiles = fs.readdirSync(agentsDir).filter(file => file.endsWith('.md')); - - for (const file of agentFiles) { - const content = fs.readFileSync(path.join(agentsDir, file), 'utf8'); - const frontmatter = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); - assert.ok(frontmatter, `${file} should have frontmatter`); - - const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m); - assert.ok(toolsLine, `${file} should declare a non-empty tools scalar`); - assert.ok( - !toolsLine[1].trim().startsWith('['), - `${file} should use comma-separated scalar tools, not a YAML sequence` - ); - } - })) passed++; else failed++; - if (test('fails on agent without frontmatter', () => { const testDir = createTestDir(); fs.writeFileSync(path.join(testDir, 'bad-agent.md'), '# No frontmatter here\nJust content.'); @@ -392,54 +374,6 @@ function runTests() { cleanupTestDir(testDir); })) passed++; else failed++; - if (test('accepts comma-separated scalar agent tools', () => { - const testDir = createTestDir(); - fs.writeFileSync(path.join(testDir, 'scalar-tools.md'), '---\nmodel: sonnet\ntools: Read, Glob, Grep\n---\n# Agent'); - - const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); - assert.strictEqual(result.code, 0, `Should accept scalar tools, got stderr: ${result.stderr}`); - cleanupTestDir(testDir); - })) passed++; else failed++; - - if (test('rejects YAML sequence-form agent tools', () => { - const testDir = createTestDir(); - fs.writeFileSync(path.join(testDir, 'sequence-tools.md'), '---\nmodel: sonnet\ntools: [Read, Glob, Grep]\n---\n# Agent'); - - const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); - assert.strictEqual(result.code, 1, 'Should reject sequence-form tools'); - assert.ok( - result.stderr.includes('comma-separated scalar'), - `Should explain the supported tools format, got stderr: ${result.stderr}` - ); - cleanupTestDir(testDir); - })) passed++; else failed++; - - if (test('rejects block sequence-form agent tools', () => { - const testDir = createTestDir(); - fs.writeFileSync(path.join(testDir, 'block-sequence-tools.md'), '---\nmodel: sonnet\ntools:\n - Read\n - Glob\n - Grep\n---\n# Agent'); - - const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); - assert.strictEqual(result.code, 1, 'Should reject block sequence-form tools'); - assert.ok( - result.stderr.includes('comma-separated scalar'), - `Should explain the supported tools format, got stderr: ${result.stderr}` - ); - cleanupTestDir(testDir); - })) passed++; else failed++; - - if (test('rejects explicitly tagged YAML sequence-form agent tools', () => { - const testDir = createTestDir(); - fs.writeFileSync(path.join(testDir, 'tagged-sequence-tools.md'), '---\nmodel: sonnet\ntools: !!seq [Read, Glob, Grep]\n---\n# Agent'); - - const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir); - assert.strictEqual(result.code, 1, 'Should reject tagged sequence-form tools'); - assert.ok( - result.stderr.includes('comma-separated scalar'), - `Should explain the supported tools format, got stderr: ${result.stderr}` - ); - cleanupTestDir(testDir); - })) passed++; else failed++; - if (test('handles frontmatter with BOM and CRLF', () => { const testDir = createTestDir(); const content = '\uFEFF---\r\nmodel: sonnet\r\ntools: Read, Write\r\n---\r\n# Agent'; diff --git a/tests/scripts/dashboard-web.test.js b/tests/scripts/dashboard-web.test.js index 304f3c921..88861b6ca 100644 --- a/tests/scripts/dashboard-web.test.js +++ b/tests/scripts/dashboard-web.test.js @@ -36,6 +36,29 @@ function cleanup(dirPath) { fs.rmSync(dirPath, { recursive: true, force: true }); } +function withTempDir(prefix, fn) { + const dirPath = createTempDir(prefix); + try { + return fn(dirPath); + } finally { + cleanup(dirPath); + } +} + +test('withTempDir removes temp directories when the callback throws', () => { + let createdDir = ''; + assert.throws(() => { + withTempDir('ecc-test-', dirPath => { + createdDir = dirPath; + assert.ok(fs.existsSync(createdDir)); + throw new Error('fixture failure'); + }); + }, /fixture failure/); + + assert.ok(createdDir); + assert.ok(!fs.existsSync(createdDir)); +}); + function writeFile(rootDir, relativePath, content) { const targetPath = path.join(rootDir, relativePath); fs.mkdirSync(path.dirname(targetPath), { recursive: true }); @@ -127,22 +150,22 @@ test('readFrontmatter parses array tools field', () => { test('readFrontmatter preserves scoped tools in legacy flow sequences', () => { const { readFrontmatter } = require(SCRIPT); - testRoot = createTempDir('ecc-test-'); - writeFile(testRoot, 'agent.md', [ - '---', - 'name: scoped-agent', - 'tools: [Agent(worker, researcher), Read, Bash(git commit:*, git status:*)]', - '---', - 'body', - ].join('\n')); + withTempDir('ecc-test-', tempDir => { + writeFile(tempDir, 'agent.md', [ + '---', + 'name: scoped-agent', + 'tools: [Agent(worker, researcher), Read, Bash(git commit:*, git status:*)]', + '---', + 'body', + ].join('\n')); - const fm = readFrontmatter(path.join(testRoot, 'agent.md')); - assert.deepStrictEqual(fm.tools, [ - 'Agent(worker, researcher)', - 'Read', - 'Bash(git commit:*, git status:*)', - ]); - cleanup(testRoot); + const fm = readFrontmatter(path.join(tempDir, 'agent.md')); + assert.deepStrictEqual(fm.tools, [ + 'Agent(worker, researcher)', + 'Read', + 'Bash(git commit:*, git status:*)', + ]); + }); }); test('readFrontmatter normalizes comma-separated scalar tools to an array', () => {