Getting ready for testing the full interpreters
This commit is contained in:
parent
727f461fb6
commit
a6cfe15a29
|
|
@ -39,3 +39,17 @@ value: `-0.25`
|
|||
This test fails because the interpreter fails to recognize a token starting with
|
||||
both a `-` then a `.` then a digit, as a numeric literal, resulting in it being
|
||||
interpreted as an identifier.
|
||||
|
||||
## TokenString Error Inside
|
||||
code: `{ 2 3a + }`
|
||||
error: `Invalid decimal literal: unexpected 'a' in decimal integer.`
|
||||
|
||||
I don't know why this test is being reported as failing. I think it may be
|
||||
trying looking for the error inside the token string.
|
||||
|
||||
## TokenString Struct Fields
|
||||
code: `{ x: y: }`
|
||||
value: `TokenString`
|
||||
|
||||
All non-code token strings won't work properly unless they are also valid code
|
||||
token strings. All non-code token string features won't be implemented yet.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
# SLS Python
|
||||
|
||||
This is the Python implementation for the YREA SLS interpreter.
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# SLS Rust
|
||||
|
||||
This is the Rust implementation for the YREA SLS interpreter.
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
tests:
|
||||
- name: "Simple print statement"
|
||||
stdin: |
|
||||
print("Hello, World!")
|
||||
stdout: |
|
||||
Hello, World!
|
||||
exit: success
|
||||
|
||||
- name: "Basic arithmetic"
|
||||
stdin: |
|
||||
print(2 + 2)
|
||||
print(10 * 5)
|
||||
stdout: |
|
||||
4
|
||||
50
|
||||
exit: success
|
||||
|
||||
- name: "Variable assignment and usage"
|
||||
stdin: |
|
||||
x = 42
|
||||
y = 8
|
||||
print(x + y)
|
||||
stdout: |
|
||||
50
|
||||
exit: success
|
||||
|
||||
- name: "Syntax error detection"
|
||||
stdin: |
|
||||
print("missing closing quote)
|
||||
stderr: |
|
||||
SyntaxError: unterminated string literal (detected at line 1)
|
||||
exit: error
|
||||
error_code: 1
|
||||
|
||||
- name: "Import and use module"
|
||||
stdin: |
|
||||
import math
|
||||
print(math.pi)
|
||||
stdout: |
|
||||
3.141592653589793
|
||||
exit: success
|
||||
|
||||
- name: "List operations"
|
||||
stdin: |
|
||||
numbers = [1, 2, 3, 4, 5]
|
||||
print(sum(numbers))
|
||||
print(len(numbers))
|
||||
stdout: |
|
||||
15
|
||||
5
|
||||
exit: success
|
||||
|
||||
- name: "Execute Python file with args"
|
||||
args: ["-c", "import sys; print(f'Args: {sys.argv[1:]}'); print('Done')", "arg1", "arg2"]
|
||||
stdout: |
|
||||
Args: ['arg1', 'arg2']
|
||||
Done
|
||||
exit: success
|
||||
|
||||
- name: "Division by zero error"
|
||||
stdin: |
|
||||
print(1 / 0)
|
||||
stderr: |
|
||||
Traceback (most recent call last):
|
||||
File "<stdin>", line 1, in <module>
|
||||
ZeroDivisionError: division by zero
|
||||
exit: error
|
||||
|
||||
- name: "Check Python version (using args instead of stdin)"
|
||||
args: ["--version"]
|
||||
stdout: "Python 3"
|
||||
exit: success
|
||||
timeout: 2.0
|
||||
|
||||
- name: "Multi-line function definition"
|
||||
stdin: |
|
||||
def greet(name):
|
||||
return f"Hello, {name}!"
|
||||
|
||||
print(greet("Alice"))
|
||||
print(greet("Bob"))
|
||||
stdout: |
|
||||
Hello, Alice!
|
||||
Hello, Bob!
|
||||
exit: success
|
||||
|
||||
- name: "Long running operation with custom timeout"
|
||||
stdin: |
|
||||
import time
|
||||
time.sleep(0.5)
|
||||
print("Done sleeping")
|
||||
stdout: |
|
||||
Done sleeping
|
||||
exit: success
|
||||
timeout: 2.0
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
CLI Testing Script - Test executables with stdin/stdout/stderr validation
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any, List
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class ExitBehavior(Enum):
|
||||
NONE = "none"
|
||||
SUCCESS = "success"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestResult:
|
||||
name: str
|
||||
passed: bool
|
||||
reason: Optional[str] = None
|
||||
|
||||
|
||||
class TestRunner:
|
||||
def __init__(self, executable: str, default_timeout: float = 5.0):
|
||||
self.executable = executable
|
||||
self.default_timeout = default_timeout
|
||||
self.results: List[TestResult] = []
|
||||
|
||||
def run_command(self, stdin: Optional[str], args: List[str], timeout: float) -> tuple:
|
||||
"""Run the executable and capture output"""
|
||||
try:
|
||||
cmd = [self.executable] + args
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
input=stdin,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout
|
||||
)
|
||||
return result.stdout, result.stderr, result.returncode, None
|
||||
except subprocess.TimeoutExpired:
|
||||
return None, None, None, "Timeout"
|
||||
except Exception as e:
|
||||
return None, None, None, f"Error: {str(e)}"
|
||||
|
||||
def normalize_output(self, text: Optional[str]) -> str:
|
||||
"""Normalize output by stripping trailing whitespace from each line"""
|
||||
if text is None:
|
||||
return ""
|
||||
lines = text.split('\n')
|
||||
return '\n'.join(line.rstrip() for line in lines)
|
||||
|
||||
def check_exit_behavior(self, returncode: int, expected: Dict[str, Any]) -> tuple[bool, Optional[str]]:
|
||||
"""Check if exit behavior matches expectation"""
|
||||
behavior = expected.get("exit", "none")
|
||||
|
||||
if behavior == "none":
|
||||
return True, None
|
||||
elif behavior == "success":
|
||||
if returncode == 0:
|
||||
return True, None
|
||||
return False, f"Expected success (exit 0), got exit code {returncode}"
|
||||
elif behavior == "error":
|
||||
expected_code = expected.get("error_code")
|
||||
if expected_code is not None:
|
||||
if returncode == expected_code:
|
||||
return True, None
|
||||
return False, f"Expected error code {expected_code}, got {returncode}"
|
||||
else:
|
||||
if returncode != 0:
|
||||
return True, None
|
||||
return False, f"Expected error (non-zero exit), got exit code 0"
|
||||
|
||||
return False, f"Unknown exit behavior: {behavior}"
|
||||
|
||||
def run_test(self, test: Dict[str, Any]) -> TestResult:
|
||||
"""Run a single test case"""
|
||||
name = test.get("name", "Unnamed test")
|
||||
timeout = test.get("timeout", self.default_timeout)
|
||||
|
||||
# Setup phase
|
||||
setup_stdin = test.get("setup")
|
||||
if setup_stdin:
|
||||
setup_args = test.get("setup_args", [])
|
||||
_, _, _, error = self.run_command(setup_stdin, setup_args, timeout)
|
||||
if error:
|
||||
return TestResult(name, False, f"Setup failed: {error}")
|
||||
|
||||
# Main test execution
|
||||
stdin = test.get("stdin")
|
||||
args = test.get("args", [])
|
||||
stdout, stderr, returncode, error = self.run_command(stdin, args, timeout)
|
||||
|
||||
if error:
|
||||
# Cleanup even on error
|
||||
self.run_cleanup(test, timeout)
|
||||
return TestResult(name, False, error)
|
||||
|
||||
# Check stdout
|
||||
expected_stdout = test.get("stdout")
|
||||
if expected_stdout is not None:
|
||||
actual = self.normalize_output(stdout)
|
||||
expected = self.normalize_output(expected_stdout)
|
||||
if actual != expected:
|
||||
self.run_cleanup(test, timeout)
|
||||
return TestResult(name, False, f"stdout mismatch\nExpected:\n{expected}\nGot:\n{actual}")
|
||||
|
||||
# Check stderr
|
||||
expected_stderr = test.get("stderr")
|
||||
if expected_stderr is not None:
|
||||
actual = self.normalize_output(stderr)
|
||||
expected = self.normalize_output(expected_stderr)
|
||||
if actual != expected:
|
||||
self.run_cleanup(test, timeout)
|
||||
return TestResult(name, False, f"stderr mismatch\nExpected:\n{expected}\nGot:\n{actual}")
|
||||
|
||||
# Check exit behavior
|
||||
passed, reason = self.check_exit_behavior(returncode, test)
|
||||
if not passed:
|
||||
self.run_cleanup(test, timeout)
|
||||
return TestResult(name, False, reason)
|
||||
|
||||
# Cleanup phase
|
||||
cleanup_result = self.run_cleanup(test, timeout)
|
||||
if cleanup_result:
|
||||
return TestResult(name, False, f"Cleanup failed: {cleanup_result}")
|
||||
|
||||
return TestResult(name, True)
|
||||
|
||||
def run_cleanup(self, test: Dict[str, Any], timeout: float) -> Optional[str]:
|
||||
"""Run cleanup if specified"""
|
||||
cleanup_stdin = test.get("cleanup")
|
||||
if cleanup_stdin:
|
||||
cleanup_args = test.get("cleanup_args", [])
|
||||
_, _, _, error = self.run_command(cleanup_stdin, cleanup_args, timeout)
|
||||
return error
|
||||
return None
|
||||
|
||||
def run_all_tests(self, tests: List[Dict[str, Any]]):
|
||||
"""Run all tests and collect results"""
|
||||
for test in tests:
|
||||
result = self.run_test(test)
|
||||
self.results.append(result)
|
||||
|
||||
# Print immediate result
|
||||
status = "✓ PASS" if result.passed else "✗ FAIL"
|
||||
print(f"{status}: {result.name}")
|
||||
if not result.passed and result.reason:
|
||||
print(f" Reason: {result.reason}")
|
||||
print()
|
||||
|
||||
def print_summary(self):
|
||||
"""Print test summary"""
|
||||
total = len(self.results)
|
||||
passed = sum(1 for r in self.results if r.passed)
|
||||
failed = total - passed
|
||||
|
||||
print("=" * 60)
|
||||
print("TEST SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total tests: {total}")
|
||||
print(f"Passed: {passed}")
|
||||
print(f"Failed: {failed}")
|
||||
|
||||
if failed > 0:
|
||||
print("\nFailed tests:")
|
||||
for result in self.results:
|
||||
if not result.passed:
|
||||
print(f" - {result.name}")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Test CLI executables with YAML test definitions")
|
||||
parser.add_argument("executable", help="Path to the executable to test")
|
||||
parser.add_argument("tests", help="Path to YAML test file")
|
||||
parser.add_argument("--timeout", type=float, default=5.0, help="Default timeout in seconds (default: 5.0)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load test file
|
||||
try:
|
||||
with open(args.tests, 'r') as f:
|
||||
test_data = yaml.safe_load(f)
|
||||
except Exception as e:
|
||||
print(f"Error loading test file: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
tests = test_data.get("tests", [])
|
||||
if not tests:
|
||||
print("No tests found in test file", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Run tests
|
||||
runner = TestRunner(args.executable, args.timeout)
|
||||
runner.run_all_tests(tests)
|
||||
|
||||
# Print summary and return exit code
|
||||
return runner.print_summary()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue