mirror of
https://github.com/SuperClaude-Org/SuperClaude_Framework.git
synced 2025-12-29 16:16:08 +00:00
feat: add comprehensive test suite, CI/CD workflows, and install command
Major improvements to SuperClaude Framework infrastructure and testing: ## New Features - Add 'superclaude install' command to install slash commands (/research, /index-repo, /agent, /recommend) - Create comprehensive test suite with 71 tests (70 passing, 1 skipped) - Add GitHub Actions CI/CD workflows for automated testing - Add essential documentation files (PLANNING.md, TASK.md, KNOWLEDGE.md) ## Testing - tests/unit/: 59 tests covering PM Agent components - test_confidence.py: 13 tests for ConfidenceChecker - test_self_check.py: 14 tests for SelfCheckProtocol - test_reflexion.py: 9 tests for ReflexionPattern - test_token_budget.py: 12 tests for TokenBudgetManager - test_cli_install.py: 12 tests for install command (NEW) - tests/integration/: 11 tests for pytest plugin integration - tests/conftest.py: Shared fixtures for all tests ## CI/CD Workflows - .github/workflows/test.yml: Comprehensive test matrix - Tests on Python 3.10, 3.11, 3.12 - Lint and format checks with ruff - Pytest plugin verification - SuperClaude doctor health checks - Coverage reporting with Codecov - .github/workflows/quick-check.yml: Fast PR validation (~2-3 min) - .github/workflows/README.md: Workflow documentation ## Documentation - PLANNING.md: Architecture, design principles, absolute rules - TASK.md: Current tasks, priorities, backlog - KNOWLEDGE.md: Accumulated insights, best practices, troubleshooting ## Bug Fixes - Fix .gitignore contradictions (remove conflicting Claude Code patterns) - Fix TokenBudgetManager to properly validate and default invalid complexity - Update package.json version to 4.1.6 (sync with VERSION file) ## CLI Improvements - src/superclaude/cli/install_commands.py: Command installation logic - src/superclaude/cli/main.py: Add 'install' command with --list and --force options - README.md: Update installation instructions with correct commands ## Breaking Changes None - all changes are backwards compatible ## Migration Guide Users should run 'superclaude install' after upgrading to install slash commands Fixes #466 (indirectly by clarifying installation process) Refs #419 (plugin system - documentation updated) 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
5
tests/unit/__init__.py
Normal file
5
tests/unit/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""
|
||||
Unit tests for SuperClaude Framework components
|
||||
|
||||
Tests individual components in isolation without external dependencies.
|
||||
"""
|
||||
181
tests/unit/test_cli_install.py
Normal file
181
tests/unit/test_cli_install.py
Normal file
@@ -0,0 +1,181 @@
|
||||
"""
|
||||
Unit tests for CLI install command
|
||||
|
||||
Tests the command installation functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from superclaude.cli.install_commands import (
|
||||
install_commands,
|
||||
list_available_commands,
|
||||
list_installed_commands,
|
||||
)
|
||||
|
||||
|
||||
class TestInstallCommands:
|
||||
"""Test suite for install commands functionality"""
|
||||
|
||||
def test_list_available_commands(self):
|
||||
"""Test listing available commands"""
|
||||
commands = list_available_commands()
|
||||
|
||||
assert isinstance(commands, list)
|
||||
assert len(commands) > 0
|
||||
assert "research" in commands
|
||||
assert "index-repo" in commands
|
||||
|
||||
def test_install_commands_to_temp_dir(self, tmp_path):
|
||||
"""Test installing commands to a temporary directory"""
|
||||
target_dir = tmp_path / "commands"
|
||||
|
||||
success, message = install_commands(target_path=target_dir, force=False)
|
||||
|
||||
assert success is True
|
||||
assert "Installed" in message
|
||||
assert target_dir.exists()
|
||||
|
||||
# Check that command files were copied
|
||||
command_files = list(target_dir.glob("*.md"))
|
||||
assert len(command_files) > 0
|
||||
|
||||
# Verify specific commands
|
||||
assert (target_dir / "research.md").exists()
|
||||
assert (target_dir / "index-repo.md").exists()
|
||||
|
||||
def test_install_commands_skip_existing(self, tmp_path):
|
||||
"""Test that existing commands are skipped without --force"""
|
||||
target_dir = tmp_path / "commands"
|
||||
|
||||
# First install
|
||||
success1, message1 = install_commands(target_path=target_dir, force=False)
|
||||
assert success1 is True
|
||||
|
||||
# Second install without force
|
||||
success2, message2 = install_commands(target_path=target_dir, force=False)
|
||||
assert success2 is True
|
||||
assert "Skipped" in message2
|
||||
|
||||
def test_install_commands_force_reinstall(self, tmp_path):
|
||||
"""Test force reinstall of existing commands"""
|
||||
target_dir = tmp_path / "commands"
|
||||
|
||||
# First install
|
||||
success1, message1 = install_commands(target_path=target_dir, force=False)
|
||||
assert success1 is True
|
||||
|
||||
# Modify a file
|
||||
research_file = target_dir / "research.md"
|
||||
research_file.write_text("modified")
|
||||
assert research_file.read_text() == "modified"
|
||||
|
||||
# Force reinstall
|
||||
success2, message2 = install_commands(target_path=target_dir, force=True)
|
||||
assert success2 is True
|
||||
assert "Installed" in message2
|
||||
|
||||
# Verify file was overwritten
|
||||
content = research_file.read_text()
|
||||
assert content != "modified"
|
||||
assert "research" in content.lower()
|
||||
|
||||
def test_list_installed_commands(self, tmp_path):
|
||||
"""Test listing installed commands"""
|
||||
target_dir = tmp_path / "commands"
|
||||
|
||||
# Before install
|
||||
# Note: list_installed_commands checks ~/.claude/commands by default
|
||||
# We can't easily test this without mocking, so just verify it returns a list
|
||||
installed = list_installed_commands()
|
||||
assert isinstance(installed, list)
|
||||
|
||||
# After install to temp dir
|
||||
install_commands(target_path=target_dir, force=False)
|
||||
|
||||
# Verify files exist
|
||||
command_files = list(target_dir.glob("*.md"))
|
||||
assert len(command_files) > 0
|
||||
|
||||
def test_install_commands_creates_target_directory(self, tmp_path):
|
||||
"""Test that target directory is created if it doesn't exist"""
|
||||
target_dir = tmp_path / "nested" / "commands"
|
||||
|
||||
assert not target_dir.exists()
|
||||
|
||||
success, message = install_commands(target_path=target_dir, force=False)
|
||||
|
||||
assert success is True
|
||||
assert target_dir.exists()
|
||||
|
||||
def test_available_commands_format(self):
|
||||
"""Test that available commands have expected format"""
|
||||
commands = list_available_commands()
|
||||
|
||||
# Should be list of strings
|
||||
assert all(isinstance(cmd, str) for cmd in commands)
|
||||
|
||||
# Should not include file extensions
|
||||
assert all(not cmd.endswith(".md") for cmd in commands)
|
||||
|
||||
# Should be sorted
|
||||
assert commands == sorted(commands)
|
||||
|
||||
def test_research_command_exists(self, tmp_path):
|
||||
"""Test that research command specifically gets installed"""
|
||||
target_dir = tmp_path / "commands"
|
||||
|
||||
install_commands(target_path=target_dir, force=False)
|
||||
|
||||
research_file = target_dir / "research.md"
|
||||
assert research_file.exists()
|
||||
|
||||
content = research_file.read_text()
|
||||
assert "research" in content.lower()
|
||||
assert len(content) > 100 # Should have substantial content
|
||||
|
||||
def test_all_expected_commands_available(self):
|
||||
"""Test that all expected commands are available"""
|
||||
commands = list_available_commands()
|
||||
|
||||
expected = ["agent", "index-repo", "recommend", "research"]
|
||||
|
||||
for expected_cmd in expected:
|
||||
assert expected_cmd in commands, f"Expected command '{expected_cmd}' not found"
|
||||
|
||||
|
||||
class TestInstallCommandsEdgeCases:
|
||||
"""Test edge cases and error handling"""
|
||||
|
||||
def test_install_to_nonexistent_parent(self, tmp_path):
|
||||
"""Test installation to path with nonexistent parent directories"""
|
||||
target_dir = tmp_path / "a" / "b" / "c" / "commands"
|
||||
|
||||
success, message = install_commands(target_path=target_dir, force=False)
|
||||
|
||||
assert success is True
|
||||
assert target_dir.exists()
|
||||
|
||||
def test_empty_target_directory_ok(self, tmp_path):
|
||||
"""Test that installation works with empty target directory"""
|
||||
target_dir = tmp_path / "commands"
|
||||
target_dir.mkdir()
|
||||
|
||||
success, message = install_commands(target_path=target_dir, force=False)
|
||||
|
||||
assert success is True
|
||||
|
||||
|
||||
def test_cli_integration():
|
||||
"""
|
||||
Integration test: verify CLI can import and use install functions
|
||||
|
||||
This tests that the CLI main.py can successfully import the functions
|
||||
"""
|
||||
from superclaude.cli.install_commands import (
|
||||
install_commands,
|
||||
list_available_commands,
|
||||
)
|
||||
|
||||
# Should not raise ImportError
|
||||
commands = list_available_commands()
|
||||
assert len(commands) > 0
|
||||
178
tests/unit/test_confidence.py
Normal file
178
tests/unit/test_confidence.py
Normal file
@@ -0,0 +1,178 @@
|
||||
"""
|
||||
Unit tests for ConfidenceChecker
|
||||
|
||||
Tests pre-execution confidence assessment functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from superclaude.pm_agent.confidence import ConfidenceChecker
|
||||
|
||||
|
||||
class TestConfidenceChecker:
|
||||
"""Test suite for ConfidenceChecker class"""
|
||||
|
||||
def test_high_confidence_scenario(self, sample_context):
|
||||
"""
|
||||
Test that a well-prepared context returns high confidence (≥90%)
|
||||
|
||||
All checks pass:
|
||||
- No duplicates (25%)
|
||||
- Architecture compliant (25%)
|
||||
- Official docs verified (20%)
|
||||
- OSS reference found (15%)
|
||||
- Root cause identified (15%)
|
||||
Total: 100%
|
||||
"""
|
||||
checker = ConfidenceChecker()
|
||||
confidence = checker.assess(sample_context)
|
||||
|
||||
assert confidence >= 0.9, f"Expected high confidence ≥0.9, got {confidence}"
|
||||
assert confidence == 1.0, "All checks passed should give 100% confidence"
|
||||
|
||||
def test_low_confidence_scenario(self, low_confidence_context):
|
||||
"""
|
||||
Test that an unprepared context returns low confidence (<70%)
|
||||
|
||||
No checks pass: 0%
|
||||
"""
|
||||
checker = ConfidenceChecker()
|
||||
confidence = checker.assess(low_confidence_context)
|
||||
|
||||
assert confidence < 0.7, f"Expected low confidence <0.7, got {confidence}"
|
||||
assert confidence == 0.0, "No checks passed should give 0% confidence"
|
||||
|
||||
def test_medium_confidence_scenario(self):
|
||||
"""
|
||||
Test medium confidence scenario (70-89%)
|
||||
|
||||
Some checks pass, some don't
|
||||
"""
|
||||
checker = ConfidenceChecker()
|
||||
context = {
|
||||
"test_name": "test_feature",
|
||||
"duplicate_check_complete": True, # 25%
|
||||
"architecture_check_complete": True, # 25%
|
||||
"official_docs_verified": True, # 20%
|
||||
"oss_reference_complete": False, # 0%
|
||||
"root_cause_identified": False, # 0%
|
||||
}
|
||||
|
||||
confidence = checker.assess(context)
|
||||
|
||||
assert 0.7 <= confidence < 0.9, f"Expected medium confidence 0.7-0.9, got {confidence}"
|
||||
assert confidence == 0.7, "Should be exactly 70%"
|
||||
|
||||
def test_confidence_checks_recorded(self, sample_context):
|
||||
"""Test that confidence checks are recorded in context"""
|
||||
checker = ConfidenceChecker()
|
||||
checker.assess(sample_context)
|
||||
|
||||
assert "confidence_checks" in sample_context
|
||||
assert isinstance(sample_context["confidence_checks"], list)
|
||||
assert len(sample_context["confidence_checks"]) == 5
|
||||
|
||||
# All checks should pass
|
||||
for check in sample_context["confidence_checks"]:
|
||||
assert check.startswith("✅"), f"Expected passing check, got: {check}"
|
||||
|
||||
def test_get_recommendation_high(self):
|
||||
"""Test recommendation for high confidence"""
|
||||
checker = ConfidenceChecker()
|
||||
recommendation = checker.get_recommendation(0.95)
|
||||
|
||||
assert "High confidence" in recommendation
|
||||
assert "Proceed" in recommendation
|
||||
|
||||
def test_get_recommendation_medium(self):
|
||||
"""Test recommendation for medium confidence"""
|
||||
checker = ConfidenceChecker()
|
||||
recommendation = checker.get_recommendation(0.75)
|
||||
|
||||
assert "Medium confidence" in recommendation
|
||||
assert "Continue investigation" in recommendation
|
||||
|
||||
def test_get_recommendation_low(self):
|
||||
"""Test recommendation for low confidence"""
|
||||
checker = ConfidenceChecker()
|
||||
recommendation = checker.get_recommendation(0.5)
|
||||
|
||||
assert "Low confidence" in recommendation
|
||||
assert "STOP" in recommendation
|
||||
|
||||
def test_has_official_docs_with_flag(self):
|
||||
"""Test official docs check with direct flag"""
|
||||
checker = ConfidenceChecker()
|
||||
context = {"official_docs_verified": True}
|
||||
|
||||
result = checker._has_official_docs(context)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_no_duplicates_check(self):
|
||||
"""Test duplicate check validation"""
|
||||
checker = ConfidenceChecker()
|
||||
|
||||
# With flag
|
||||
context_pass = {"duplicate_check_complete": True}
|
||||
assert checker._no_duplicates(context_pass) is True
|
||||
|
||||
# Without flag
|
||||
context_fail = {"duplicate_check_complete": False}
|
||||
assert checker._no_duplicates(context_fail) is False
|
||||
|
||||
def test_architecture_compliance_check(self):
|
||||
"""Test architecture compliance validation"""
|
||||
checker = ConfidenceChecker()
|
||||
|
||||
# With flag
|
||||
context_pass = {"architecture_check_complete": True}
|
||||
assert checker._architecture_compliant(context_pass) is True
|
||||
|
||||
# Without flag
|
||||
context_fail = {}
|
||||
assert checker._architecture_compliant(context_fail) is False
|
||||
|
||||
def test_oss_reference_check(self):
|
||||
"""Test OSS reference validation"""
|
||||
checker = ConfidenceChecker()
|
||||
|
||||
# With flag
|
||||
context_pass = {"oss_reference_complete": True}
|
||||
assert checker._has_oss_reference(context_pass) is True
|
||||
|
||||
# Without flag
|
||||
context_fail = {"oss_reference_complete": False}
|
||||
assert checker._has_oss_reference(context_fail) is False
|
||||
|
||||
def test_root_cause_check(self):
|
||||
"""Test root cause identification validation"""
|
||||
checker = ConfidenceChecker()
|
||||
|
||||
# With flag
|
||||
context_pass = {"root_cause_identified": True}
|
||||
assert checker._root_cause_identified(context_pass) is True
|
||||
|
||||
# Without flag
|
||||
context_fail = {}
|
||||
assert checker._root_cause_identified(context_fail) is False
|
||||
|
||||
|
||||
@pytest.mark.confidence_check
|
||||
def test_confidence_check_marker_integration(confidence_checker):
|
||||
"""
|
||||
Test that confidence_check marker works with pytest plugin fixture
|
||||
|
||||
This test should skip if confidence < 70%
|
||||
"""
|
||||
context = {
|
||||
"test_name": "test_confidence_check_marker_integration",
|
||||
"has_official_docs": True,
|
||||
"duplicate_check_complete": True,
|
||||
"architecture_check_complete": True,
|
||||
"official_docs_verified": True,
|
||||
"oss_reference_complete": True,
|
||||
"root_cause_identified": True,
|
||||
}
|
||||
|
||||
confidence = confidence_checker.assess(context)
|
||||
assert confidence >= 0.7, "Confidence should be high enough to not skip"
|
||||
182
tests/unit/test_reflexion.py
Normal file
182
tests/unit/test_reflexion.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""
|
||||
Unit tests for ReflexionPattern
|
||||
|
||||
Tests error learning and prevention functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from superclaude.pm_agent.reflexion import ReflexionPattern
|
||||
|
||||
|
||||
class TestReflexionPattern:
|
||||
"""Test suite for ReflexionPattern class"""
|
||||
|
||||
def test_initialization(self):
|
||||
"""Test ReflexionPattern initialization"""
|
||||
reflexion = ReflexionPattern()
|
||||
|
||||
assert reflexion is not None
|
||||
assert hasattr(reflexion, "record_error")
|
||||
assert hasattr(reflexion, "get_solution")
|
||||
|
||||
def test_record_error_basic(self):
|
||||
"""Test recording a basic error"""
|
||||
reflexion = ReflexionPattern()
|
||||
|
||||
error_info = {
|
||||
"test_name": "test_feature",
|
||||
"error_type": "AssertionError",
|
||||
"error_message": "Expected 5, got 3",
|
||||
"traceback": "File test.py, line 10...",
|
||||
}
|
||||
|
||||
# Should not raise an exception
|
||||
reflexion.record_error(error_info)
|
||||
|
||||
def test_record_error_with_solution(self):
|
||||
"""Test recording an error with a solution"""
|
||||
reflexion = ReflexionPattern()
|
||||
|
||||
error_info = {
|
||||
"test_name": "test_database_connection",
|
||||
"error_type": "ConnectionError",
|
||||
"error_message": "Could not connect to database",
|
||||
"solution": "Ensure database is running and credentials are correct",
|
||||
}
|
||||
|
||||
reflexion.record_error(error_info)
|
||||
|
||||
def test_get_solution_for_known_error(self):
|
||||
"""Test retrieving solution for a known error pattern"""
|
||||
reflexion = ReflexionPattern()
|
||||
|
||||
# Record an error with solution
|
||||
error_info = {
|
||||
"error_type": "ImportError",
|
||||
"error_message": "No module named 'pytest'",
|
||||
"solution": "Install pytest: pip install pytest",
|
||||
}
|
||||
|
||||
reflexion.record_error(error_info)
|
||||
|
||||
# Try to get solution for similar error
|
||||
error_signature = "ImportError: No module named 'pytest'"
|
||||
solution = reflexion.get_solution(error_signature)
|
||||
|
||||
# Note: Actual implementation might return None if not implemented yet
|
||||
# This test documents expected behavior
|
||||
assert solution is None or isinstance(solution, str)
|
||||
|
||||
def test_error_pattern_matching(self):
|
||||
"""Test error pattern matching functionality"""
|
||||
reflexion = ReflexionPattern()
|
||||
|
||||
# Record multiple similar errors
|
||||
errors = [
|
||||
{
|
||||
"error_type": "TypeError",
|
||||
"error_message": "expected str, got int",
|
||||
"solution": "Convert int to str using str()",
|
||||
},
|
||||
{
|
||||
"error_type": "TypeError",
|
||||
"error_message": "expected int, got str",
|
||||
"solution": "Convert str to int using int()",
|
||||
},
|
||||
]
|
||||
|
||||
for error in errors:
|
||||
reflexion.record_error(error)
|
||||
|
||||
# Test pattern matching (implementation-dependent)
|
||||
error_signature = "TypeError"
|
||||
solution = reflexion.get_solution(error_signature)
|
||||
|
||||
assert solution is None or isinstance(solution, str)
|
||||
|
||||
def test_reflexion_memory_persistence(self, temp_memory_dir):
|
||||
"""Test that reflexion can work with memory directory"""
|
||||
reflexion = ReflexionPattern(memory_dir=temp_memory_dir)
|
||||
|
||||
error_info = {
|
||||
"test_name": "test_feature",
|
||||
"error_type": "ValueError",
|
||||
"error_message": "Invalid input",
|
||||
}
|
||||
|
||||
# Should not raise exception even with custom memory dir
|
||||
reflexion.record_error(error_info)
|
||||
|
||||
def test_error_learning_across_sessions(self):
|
||||
"""
|
||||
Test that errors can be learned across sessions
|
||||
|
||||
Note: This tests the interface, actual persistence
|
||||
depends on implementation
|
||||
"""
|
||||
reflexion = ReflexionPattern()
|
||||
|
||||
# Session 1: Record error
|
||||
error_info = {
|
||||
"error_type": "FileNotFoundError",
|
||||
"error_message": "config.json not found",
|
||||
"solution": "Create config.json in project root",
|
||||
"session": "session_1",
|
||||
}
|
||||
|
||||
reflexion.record_error(error_info)
|
||||
|
||||
# Session 2: Retrieve solution
|
||||
error_signature = "FileNotFoundError: config.json"
|
||||
solution = reflexion.get_solution(error_signature)
|
||||
|
||||
# Implementation may or may not persist across instances
|
||||
assert solution is None or isinstance(solution, str)
|
||||
|
||||
|
||||
@pytest.mark.reflexion
|
||||
def test_reflexion_marker_integration(reflexion_pattern):
|
||||
"""
|
||||
Test that reflexion marker works with pytest plugin fixture
|
||||
|
||||
If this test fails, reflexion should record the failure
|
||||
"""
|
||||
# Test that fixture is properly provided
|
||||
assert reflexion_pattern is not None
|
||||
|
||||
# Record a test error
|
||||
error_info = {
|
||||
"test_name": "test_reflexion_marker_integration",
|
||||
"error_type": "IntegrationTestError",
|
||||
"error_message": "Testing reflexion integration",
|
||||
}
|
||||
|
||||
# Should not raise exception
|
||||
reflexion_pattern.record_error(error_info)
|
||||
|
||||
|
||||
def test_reflexion_with_real_exception():
|
||||
"""
|
||||
Test reflexion pattern with a real exception scenario
|
||||
|
||||
This simulates how reflexion would be used in practice
|
||||
"""
|
||||
reflexion = ReflexionPattern()
|
||||
|
||||
try:
|
||||
# Simulate an operation that fails
|
||||
result = 10 / 0
|
||||
except ZeroDivisionError as e:
|
||||
# Record the error
|
||||
error_info = {
|
||||
"test_name": "test_reflexion_with_real_exception",
|
||||
"error_type": type(e).__name__,
|
||||
"error_message": str(e),
|
||||
"traceback": "simulated traceback",
|
||||
"solution": "Check denominator is not zero before division",
|
||||
}
|
||||
|
||||
reflexion.record_error(error_info)
|
||||
|
||||
# Test should complete successfully
|
||||
assert True
|
||||
235
tests/unit/test_self_check.py
Normal file
235
tests/unit/test_self_check.py
Normal file
@@ -0,0 +1,235 @@
|
||||
"""
|
||||
Unit tests for SelfCheckProtocol
|
||||
|
||||
Tests post-implementation validation functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from superclaude.pm_agent.self_check import SelfCheckProtocol
|
||||
|
||||
|
||||
class TestSelfCheckProtocol:
|
||||
"""Test suite for SelfCheckProtocol class"""
|
||||
|
||||
def test_validate_passing_implementation(self, sample_implementation):
|
||||
"""
|
||||
Test validation of a complete, passing implementation
|
||||
|
||||
Should pass all four questions:
|
||||
1. Tests passing? ✅
|
||||
2. Requirements met? ✅
|
||||
3. Assumptions verified? ✅
|
||||
4. Evidence provided? ✅
|
||||
"""
|
||||
protocol = SelfCheckProtocol()
|
||||
passed, issues = protocol.validate(sample_implementation)
|
||||
|
||||
assert passed is True, f"Expected validation to pass, got issues: {issues}"
|
||||
assert len(issues) == 0, f"Expected no issues, got {len(issues)}: {issues}"
|
||||
|
||||
def test_validate_failing_implementation(self, failing_implementation):
|
||||
"""
|
||||
Test validation of a failing implementation
|
||||
|
||||
Should fail multiple checks
|
||||
"""
|
||||
protocol = SelfCheckProtocol()
|
||||
passed, issues = protocol.validate(failing_implementation)
|
||||
|
||||
assert passed is False, "Expected validation to fail"
|
||||
assert len(issues) > 0, "Expected issues to be detected"
|
||||
|
||||
# Check specific issues
|
||||
issue_text = " ".join(issues)
|
||||
assert "Tests not passing" in issue_text or "test" in issue_text.lower()
|
||||
|
||||
def test_check_tests_passing_with_output(self):
|
||||
"""Test that tests_passed requires actual output"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
# Tests passed WITH output - should pass
|
||||
impl_with_output = {
|
||||
"tests_passed": True,
|
||||
"test_output": "✅ 10 tests passed",
|
||||
}
|
||||
assert protocol._check_tests_passing(impl_with_output) is True
|
||||
|
||||
# Tests passed WITHOUT output - should fail (hallucination detection)
|
||||
impl_without_output = {
|
||||
"tests_passed": True,
|
||||
"test_output": "",
|
||||
}
|
||||
assert protocol._check_tests_passing(impl_without_output) is False
|
||||
|
||||
def test_check_requirements_met(self):
|
||||
"""Test requirements validation"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
# All requirements met
|
||||
impl_complete = {
|
||||
"requirements": ["A", "B", "C"],
|
||||
"requirements_met": ["A", "B", "C"],
|
||||
}
|
||||
unmet = protocol._check_requirements_met(impl_complete)
|
||||
assert len(unmet) == 0
|
||||
|
||||
# Some requirements not met
|
||||
impl_incomplete = {
|
||||
"requirements": ["A", "B", "C"],
|
||||
"requirements_met": ["A", "B"],
|
||||
}
|
||||
unmet = protocol._check_requirements_met(impl_incomplete)
|
||||
assert len(unmet) == 1
|
||||
assert "C" in unmet
|
||||
|
||||
def test_check_assumptions_verified(self):
|
||||
"""Test assumptions verification"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
# All assumptions verified
|
||||
impl_verified = {
|
||||
"assumptions": ["API is REST", "DB is PostgreSQL"],
|
||||
"assumptions_verified": ["API is REST", "DB is PostgreSQL"],
|
||||
}
|
||||
unverified = protocol._check_assumptions_verified(impl_verified)
|
||||
assert len(unverified) == 0
|
||||
|
||||
# Some assumptions unverified
|
||||
impl_unverified = {
|
||||
"assumptions": ["API is REST", "DB is PostgreSQL"],
|
||||
"assumptions_verified": ["API is REST"],
|
||||
}
|
||||
unverified = protocol._check_assumptions_verified(impl_unverified)
|
||||
assert len(unverified) == 1
|
||||
assert "DB is PostgreSQL" in unverified
|
||||
|
||||
def test_check_evidence_exists(self):
|
||||
"""Test evidence requirement validation"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
# All evidence present
|
||||
impl_with_evidence = {
|
||||
"evidence": {
|
||||
"test_results": "Tests passed",
|
||||
"code_changes": ["file1.py"],
|
||||
"validation": "Linting passed",
|
||||
}
|
||||
}
|
||||
missing = protocol._check_evidence_exists(impl_with_evidence)
|
||||
assert len(missing) == 0
|
||||
|
||||
# Missing all evidence
|
||||
impl_no_evidence = {"evidence": {}}
|
||||
missing = protocol._check_evidence_exists(impl_no_evidence)
|
||||
assert len(missing) == 3
|
||||
assert "test_results" in missing
|
||||
assert "code_changes" in missing
|
||||
assert "validation" in missing
|
||||
|
||||
def test_detect_hallucinations_tests_without_output(self):
|
||||
"""Test hallucination detection: claims tests pass without output"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
impl = {
|
||||
"tests_passed": True,
|
||||
"test_output": "", # No output - hallucination!
|
||||
}
|
||||
|
||||
detected = protocol._detect_hallucinations(impl)
|
||||
|
||||
assert len(detected) > 0
|
||||
assert any("without showing output" in d for d in detected)
|
||||
|
||||
def test_detect_hallucinations_complete_without_evidence(self):
|
||||
"""Test hallucination detection: claims complete without evidence"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
impl = {
|
||||
"status": "complete",
|
||||
"evidence": None, # No evidence - hallucination!
|
||||
}
|
||||
|
||||
detected = protocol._detect_hallucinations(impl)
|
||||
|
||||
assert len(detected) > 0
|
||||
assert any("without evidence" in d for d in detected)
|
||||
|
||||
def test_detect_hallucinations_complete_with_failing_tests(self):
|
||||
"""Test hallucination detection: claims complete despite failing tests"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
impl = {
|
||||
"status": "complete",
|
||||
"tests_passed": False, # Tests failed but claims complete!
|
||||
}
|
||||
|
||||
detected = protocol._detect_hallucinations(impl)
|
||||
|
||||
assert len(detected) > 0
|
||||
assert any("failing tests" in d for d in detected)
|
||||
|
||||
def test_detect_hallucinations_ignored_errors(self):
|
||||
"""Test hallucination detection: ignored errors/warnings"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
impl = {
|
||||
"status": "complete",
|
||||
"errors": ["TypeError in module X"],
|
||||
"warnings": ["Deprecated function used"],
|
||||
}
|
||||
|
||||
detected = protocol._detect_hallucinations(impl)
|
||||
|
||||
assert len(detected) > 0
|
||||
assert any("errors/warnings" in d for d in detected)
|
||||
|
||||
def test_detect_hallucinations_uncertainty_language(self):
|
||||
"""Test hallucination detection: uncertainty language"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
impl = {
|
||||
"description": "This probably works and might be correct",
|
||||
}
|
||||
|
||||
detected = protocol._detect_hallucinations(impl)
|
||||
|
||||
assert len(detected) > 0
|
||||
assert any("Uncertainty language" in d for d in detected)
|
||||
|
||||
def test_format_report_passing(self):
|
||||
"""Test report formatting for passing validation"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
report = protocol.format_report(passed=True, issues=[])
|
||||
|
||||
assert "PASSED" in report
|
||||
assert "✅" in report
|
||||
|
||||
def test_format_report_failing(self):
|
||||
"""Test report formatting for failing validation"""
|
||||
protocol = SelfCheckProtocol()
|
||||
|
||||
issues = [
|
||||
"❌ Tests not passing",
|
||||
"❌ Missing evidence: test_results",
|
||||
]
|
||||
|
||||
report = protocol.format_report(passed=False, issues=issues)
|
||||
|
||||
assert "FAILED" in report
|
||||
assert "❌" in report
|
||||
for issue in issues:
|
||||
assert issue in report
|
||||
|
||||
|
||||
@pytest.mark.self_check
|
||||
def test_self_check_marker_integration(self_check_protocol, sample_implementation):
|
||||
"""
|
||||
Test that self_check marker works with pytest plugin fixture
|
||||
|
||||
This test validates the fixture provided by pytest plugin
|
||||
"""
|
||||
passed, issues = self_check_protocol.validate(sample_implementation)
|
||||
|
||||
assert passed is True, f"Sample implementation should pass validation: {issues}"
|
||||
assert len(issues) == 0, "No issues should be detected in sample implementation"
|
||||
128
tests/unit/test_token_budget.py
Normal file
128
tests/unit/test_token_budget.py
Normal file
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
Unit tests for TokenBudgetManager
|
||||
|
||||
Tests token budget allocation and management functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from superclaude.pm_agent.token_budget import TokenBudgetManager
|
||||
|
||||
|
||||
class TestTokenBudgetManager:
|
||||
"""Test suite for TokenBudgetManager class"""
|
||||
|
||||
def test_simple_complexity(self):
|
||||
"""Test token budget for simple tasks (typo fixes)"""
|
||||
manager = TokenBudgetManager(complexity="simple")
|
||||
|
||||
assert manager.limit == 200
|
||||
assert manager.complexity == "simple"
|
||||
|
||||
def test_medium_complexity(self):
|
||||
"""Test token budget for medium tasks (bug fixes)"""
|
||||
manager = TokenBudgetManager(complexity="medium")
|
||||
|
||||
assert manager.limit == 1000
|
||||
assert manager.complexity == "medium"
|
||||
|
||||
def test_complex_complexity(self):
|
||||
"""Test token budget for complex tasks (features)"""
|
||||
manager = TokenBudgetManager(complexity="complex")
|
||||
|
||||
assert manager.limit == 2500
|
||||
assert manager.complexity == "complex"
|
||||
|
||||
def test_default_complexity(self):
|
||||
"""Test default complexity is medium"""
|
||||
manager = TokenBudgetManager()
|
||||
|
||||
assert manager.limit == 1000
|
||||
assert manager.complexity == "medium"
|
||||
|
||||
def test_invalid_complexity_defaults_to_medium(self):
|
||||
"""Test that invalid complexity defaults to medium"""
|
||||
manager = TokenBudgetManager(complexity="invalid")
|
||||
|
||||
assert manager.limit == 1000
|
||||
assert manager.complexity == "medium"
|
||||
|
||||
def test_token_usage_tracking(self):
|
||||
"""Test token usage tracking if implemented"""
|
||||
manager = TokenBudgetManager(complexity="simple")
|
||||
|
||||
# Check if usage tracking is available
|
||||
if hasattr(manager, "used"):
|
||||
assert manager.used == 0
|
||||
|
||||
if hasattr(manager, "remaining"):
|
||||
assert manager.remaining == manager.limit
|
||||
|
||||
def test_budget_allocation_strategy(self):
|
||||
"""Test token budget allocation strategy"""
|
||||
# Simple tasks should have smallest budget
|
||||
simple = TokenBudgetManager(complexity="simple")
|
||||
|
||||
# Medium tasks should have moderate budget
|
||||
medium = TokenBudgetManager(complexity="medium")
|
||||
|
||||
# Complex tasks should have largest budget
|
||||
complex_task = TokenBudgetManager(complexity="complex")
|
||||
|
||||
assert simple.limit < medium.limit < complex_task.limit
|
||||
|
||||
def test_complexity_examples(self):
|
||||
"""Test that complexity levels match documented examples"""
|
||||
# Simple: typo fix (200 tokens)
|
||||
simple = TokenBudgetManager(complexity="simple")
|
||||
assert simple.limit == 200
|
||||
|
||||
# Medium: bug fix, small feature (1,000 tokens)
|
||||
medium = TokenBudgetManager(complexity="medium")
|
||||
assert medium.limit == 1000
|
||||
|
||||
# Complex: feature implementation (2,500 tokens)
|
||||
complex_task = TokenBudgetManager(complexity="complex")
|
||||
assert complex_task.limit == 2500
|
||||
|
||||
|
||||
@pytest.mark.complexity("simple")
|
||||
def test_complexity_marker_simple(token_budget):
|
||||
"""
|
||||
Test that complexity marker works with pytest plugin fixture
|
||||
|
||||
This test should have a simple (200 token) budget
|
||||
"""
|
||||
assert token_budget.limit == 200
|
||||
assert token_budget.complexity == "simple"
|
||||
|
||||
|
||||
@pytest.mark.complexity("medium")
|
||||
def test_complexity_marker_medium(token_budget):
|
||||
"""
|
||||
Test that complexity marker works with medium budget
|
||||
|
||||
This test should have a medium (1000 token) budget
|
||||
"""
|
||||
assert token_budget.limit == 1000
|
||||
assert token_budget.complexity == "medium"
|
||||
|
||||
|
||||
@pytest.mark.complexity("complex")
|
||||
def test_complexity_marker_complex(token_budget):
|
||||
"""
|
||||
Test that complexity marker works with complex budget
|
||||
|
||||
This test should have a complex (2500 token) budget
|
||||
"""
|
||||
assert token_budget.limit == 2500
|
||||
assert token_budget.complexity == "complex"
|
||||
|
||||
|
||||
def test_token_budget_no_marker(token_budget):
|
||||
"""
|
||||
Test that token_budget fixture defaults to medium without marker
|
||||
|
||||
Tests without complexity marker should get medium budget
|
||||
"""
|
||||
assert token_budget.limit == 1000
|
||||
assert token_budget.complexity == "medium"
|
||||
Reference in New Issue
Block a user