#!/usr/bin/env python3 """ CLI Testing Script - Test executables with stdin/stdout/stderr validation """ import argparse import subprocess import sys import yaml from pathlib import Path from typing import Optional, Dict, Any, List from dataclasses import dataclass from enum import Enum class ExitBehavior(Enum): NONE = "none" SUCCESS = "success" ERROR = "error" @dataclass class TestResult: name: str passed: bool reason: Optional[str] = None class TestRunner: def __init__(self, executable: str, default_timeout: float = 5.0): self.executable = executable self.default_timeout = default_timeout self.results: List[TestResult] = [] def run_command(self, stdin: Optional[str], args: List[str], timeout: float) -> tuple: """Run the executable and capture output""" try: cmd = [self.executable] + args result = subprocess.run( cmd, input=stdin, capture_output=True, text=True, timeout=timeout ) return result.stdout, result.stderr, result.returncode, None except subprocess.TimeoutExpired: return None, None, None, "Timeout" except Exception as e: return None, None, None, f"Error: {str(e)}" def normalize_output(self, text: Optional[str]) -> str: """Normalize output by stripping trailing whitespace from each line""" if text is None: return "" lines = text.split('\n') return '\n'.join(line.rstrip() for line in lines) def check_exit_behavior(self, returncode: int, expected: Dict[str, Any]) -> tuple[bool, Optional[str]]: """Check if exit behavior matches expectation""" behavior = expected.get("exit", "none") if behavior == "none": return True, None elif behavior == "success": if returncode == 0: return True, None return False, f"Expected success (exit 0), got exit code {returncode}" elif behavior == "error": expected_code = expected.get("error_code") if expected_code is not None: if returncode == expected_code: return True, None return False, f"Expected error code {expected_code}, got {returncode}" else: if returncode != 0: return True, None return False, f"Expected error (non-zero exit), got exit code 0" return False, f"Unknown exit behavior: {behavior}" def run_test(self, test: Dict[str, Any]) -> TestResult: """Run a single test case""" name = test.get("name", "Unnamed test") timeout = test.get("timeout", self.default_timeout) # Setup phase setup_stdin = test.get("setup") if setup_stdin: setup_args = test.get("setup_args", []) _, _, _, error = self.run_command(setup_stdin, setup_args, timeout) if error: return TestResult(name, False, f"Setup failed: {error}") # Main test execution stdin = test.get("stdin") args = test.get("args", []) stdout, stderr, returncode, error = self.run_command(stdin, args, timeout) if error: # Cleanup even on error self.run_cleanup(test, timeout) return TestResult(name, False, error) # Check stdout expected_stdout = test.get("stdout") if expected_stdout is not None: actual = self.normalize_output(stdout) expected = self.normalize_output(expected_stdout) if actual != expected: self.run_cleanup(test, timeout) return TestResult(name, False, f"stdout mismatch\nExpected:\n{expected}\nGot:\n{actual}") # Check stderr expected_stderr = test.get("stderr") if expected_stderr is not None: actual = self.normalize_output(stderr) expected = self.normalize_output(expected_stderr) if actual != expected: self.run_cleanup(test, timeout) return TestResult(name, False, f"stderr mismatch\nExpected:\n{expected}\nGot:\n{actual}") # Check exit behavior passed, reason = self.check_exit_behavior(returncode, test) if not passed: self.run_cleanup(test, timeout) return TestResult(name, False, reason) # Cleanup phase cleanup_result = self.run_cleanup(test, timeout) if cleanup_result: return TestResult(name, False, f"Cleanup failed: {cleanup_result}") return TestResult(name, True) def run_cleanup(self, test: Dict[str, Any], timeout: float) -> Optional[str]: """Run cleanup if specified""" cleanup_stdin = test.get("cleanup") if cleanup_stdin: cleanup_args = test.get("cleanup_args", []) _, _, _, error = self.run_command(cleanup_stdin, cleanup_args, timeout) return error return None def run_all_tests(self, tests: List[Dict[str, Any]]): """Run all tests and collect results""" for test in tests: result = self.run_test(test) self.results.append(result) # Print immediate result status = "✓ PASS" if result.passed else "✗ FAIL" print(f"{status}: {result.name}") if not result.passed and result.reason: print(f" Reason: {result.reason}") print() def print_summary(self): """Print test summary""" total = len(self.results) passed = sum(1 for r in self.results if r.passed) failed = total - passed print("=" * 60) print("TEST SUMMARY") print("=" * 60) print(f"Total tests: {total}") print(f"Passed: {passed}") print(f"Failed: {failed}") if failed > 0: print("\nFailed tests:") for result in self.results: if not result.passed: print(f" - {result.name}") print("=" * 60) return 0 if failed == 0 else 1 def main(): parser = argparse.ArgumentParser(description="Test CLI executables with YAML test definitions") parser.add_argument("executable", help="Path to the executable to test") parser.add_argument("tests", help="Path to YAML test file") parser.add_argument("--timeout", type=float, default=5.0, help="Default timeout in seconds (default: 5.0)") args = parser.parse_args() # Load test file try: with open(args.tests, 'r') as f: test_data = yaml.safe_load(f) except Exception as e: print(f"Error loading test file: {e}", file=sys.stderr) return 1 tests = test_data.get("tests", []) if not tests: print("No tests found in test file", file=sys.stderr) return 1 # Run tests runner = TestRunner(args.executable, args.timeout) runner.run_all_tests(tests) # Print summary and return exit code return runner.print_summary() if __name__ == "__main__": sys.exit(main())