213 lines
7.2 KiB
Python
213 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
CLI Testing Script - Test executables with stdin/stdout/stderr validation
|
|
"""
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
import yaml
|
|
from pathlib import Path
|
|
from typing import Optional, Dict, Any, List
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
|
|
|
|
class ExitBehavior(Enum):
|
|
NONE = "none"
|
|
SUCCESS = "success"
|
|
ERROR = "error"
|
|
|
|
|
|
@dataclass
|
|
class TestResult:
|
|
name: str
|
|
passed: bool
|
|
reason: Optional[str] = None
|
|
|
|
|
|
class TestRunner:
|
|
def __init__(self, executable: str, default_timeout: float = 5.0):
|
|
self.executable = executable
|
|
self.default_timeout = default_timeout
|
|
self.results: List[TestResult] = []
|
|
|
|
def run_command(self, stdin: Optional[str], args: List[str], timeout: float) -> tuple:
|
|
"""Run the executable and capture output"""
|
|
try:
|
|
cmd = [self.executable] + args
|
|
result = subprocess.run(
|
|
cmd,
|
|
input=stdin,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout
|
|
)
|
|
return result.stdout, result.stderr, result.returncode, None
|
|
except subprocess.TimeoutExpired:
|
|
return None, None, None, "Timeout"
|
|
except Exception as e:
|
|
return None, None, None, f"Error: {str(e)}"
|
|
|
|
def normalize_output(self, text: Optional[str]) -> str:
|
|
"""Normalize output by stripping trailing whitespace from each line"""
|
|
if text is None:
|
|
return ""
|
|
lines = text.split('\n')
|
|
return '\n'.join(line.rstrip() for line in lines)
|
|
|
|
def check_exit_behavior(self, returncode: int, expected: Dict[str, Any]) -> tuple[bool, Optional[str]]:
|
|
"""Check if exit behavior matches expectation"""
|
|
behavior = expected.get("exit", "none")
|
|
|
|
if behavior == "none":
|
|
return True, None
|
|
elif behavior == "success":
|
|
if returncode == 0:
|
|
return True, None
|
|
return False, f"Expected success (exit 0), got exit code {returncode}"
|
|
elif behavior == "error":
|
|
expected_code = expected.get("error_code")
|
|
if expected_code is not None:
|
|
if returncode == expected_code:
|
|
return True, None
|
|
return False, f"Expected error code {expected_code}, got {returncode}"
|
|
else:
|
|
if returncode != 0:
|
|
return True, None
|
|
return False, f"Expected error (non-zero exit), got exit code 0"
|
|
|
|
return False, f"Unknown exit behavior: {behavior}"
|
|
|
|
def run_test(self, test: Dict[str, Any]) -> TestResult:
|
|
"""Run a single test case"""
|
|
name = test.get("name", "Unnamed test")
|
|
timeout = test.get("timeout", self.default_timeout)
|
|
|
|
# Setup phase
|
|
setup_stdin = test.get("setup")
|
|
if setup_stdin:
|
|
setup_args = test.get("setup_args", [])
|
|
_, _, _, error = self.run_command(setup_stdin, setup_args, timeout)
|
|
if error:
|
|
return TestResult(name, False, f"Setup failed: {error}")
|
|
|
|
# Main test execution
|
|
stdin = test.get("stdin")
|
|
args = test.get("args", [])
|
|
stdout, stderr, returncode, error = self.run_command(stdin, args, timeout)
|
|
|
|
if error:
|
|
# Cleanup even on error
|
|
self.run_cleanup(test, timeout)
|
|
return TestResult(name, False, error)
|
|
|
|
# Check stdout
|
|
expected_stdout = test.get("stdout")
|
|
if expected_stdout is not None:
|
|
actual = self.normalize_output(stdout)
|
|
expected = self.normalize_output(expected_stdout)
|
|
if actual != expected:
|
|
self.run_cleanup(test, timeout)
|
|
return TestResult(name, False, f"stdout mismatch\nExpected:\n{expected}\nGot:\n{actual}")
|
|
|
|
# Check stderr
|
|
expected_stderr = test.get("stderr")
|
|
if expected_stderr is not None:
|
|
actual = self.normalize_output(stderr)
|
|
expected = self.normalize_output(expected_stderr)
|
|
if actual != expected:
|
|
self.run_cleanup(test, timeout)
|
|
return TestResult(name, False, f"stderr mismatch\nExpected:\n{expected}\nGot:\n{actual}")
|
|
|
|
# Check exit behavior
|
|
passed, reason = self.check_exit_behavior(returncode, test)
|
|
if not passed:
|
|
self.run_cleanup(test, timeout)
|
|
return TestResult(name, False, reason)
|
|
|
|
# Cleanup phase
|
|
cleanup_result = self.run_cleanup(test, timeout)
|
|
if cleanup_result:
|
|
return TestResult(name, False, f"Cleanup failed: {cleanup_result}")
|
|
|
|
return TestResult(name, True)
|
|
|
|
def run_cleanup(self, test: Dict[str, Any], timeout: float) -> Optional[str]:
|
|
"""Run cleanup if specified"""
|
|
cleanup_stdin = test.get("cleanup")
|
|
if cleanup_stdin:
|
|
cleanup_args = test.get("cleanup_args", [])
|
|
_, _, _, error = self.run_command(cleanup_stdin, cleanup_args, timeout)
|
|
return error
|
|
return None
|
|
|
|
def run_all_tests(self, tests: List[Dict[str, Any]]):
|
|
"""Run all tests and collect results"""
|
|
for test in tests:
|
|
result = self.run_test(test)
|
|
self.results.append(result)
|
|
|
|
# Print immediate result
|
|
status = "✓ PASS" if result.passed else "✗ FAIL"
|
|
print(f"{status}: {result.name}")
|
|
if not result.passed and result.reason:
|
|
print(f" Reason: {result.reason}")
|
|
print()
|
|
|
|
def print_summary(self):
|
|
"""Print test summary"""
|
|
total = len(self.results)
|
|
passed = sum(1 for r in self.results if r.passed)
|
|
failed = total - passed
|
|
|
|
print("=" * 60)
|
|
print("TEST SUMMARY")
|
|
print("=" * 60)
|
|
print(f"Total tests: {total}")
|
|
print(f"Passed: {passed}")
|
|
print(f"Failed: {failed}")
|
|
|
|
if failed > 0:
|
|
print("\nFailed tests:")
|
|
for result in self.results:
|
|
if not result.passed:
|
|
print(f" - {result.name}")
|
|
|
|
print("=" * 60)
|
|
|
|
return 0 if failed == 0 else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Test CLI executables with YAML test definitions")
|
|
parser.add_argument("executable", help="Path to the executable to test")
|
|
parser.add_argument("tests", help="Path to YAML test file")
|
|
parser.add_argument("--timeout", type=float, default=5.0, help="Default timeout in seconds (default: 5.0)")
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Load test file
|
|
try:
|
|
with open(args.tests, 'r') as f:
|
|
test_data = yaml.safe_load(f)
|
|
except Exception as e:
|
|
print(f"Error loading test file: {e}", file=sys.stderr)
|
|
return 1
|
|
|
|
tests = test_data.get("tests", [])
|
|
if not tests:
|
|
print("No tests found in test file", file=sys.stderr)
|
|
return 1
|
|
|
|
# Run tests
|
|
runner = TestRunner(args.executable, args.timeout)
|
|
runner.run_all_tests(tests)
|
|
|
|
# Print summary and return exit code
|
|
return runner.print_summary()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|