diff --git a/SLS_Tests/generate_tests/__init__.py b/SLS_Tests/generate_tests/__init__.py index 135eab2..0ed2997 100644 --- a/SLS_Tests/generate_tests/__init__.py +++ b/SLS_Tests/generate_tests/__init__.py @@ -5,6 +5,7 @@ from .float_tests import FloatTestGenerator from .char_tests import CharTestGenerator # from .string_tests import StringTestGenerator from .idents_and_bools_tests import IdentifierTestGenerator, BooleanTestGenerator +from .token_strings import TokenStringTestGenerator __all__ = [ "BaseTestGenerator", @@ -15,4 +16,5 @@ __all__ = [ # "StringTestGenerator", "IdentifierTestGenerator", "BooleanTestGenerator", + "TokenStringTestGenerator", ] diff --git a/SLS_Tests/generate_tests/token_strings.py b/SLS_Tests/generate_tests/token_strings.py new file mode 100644 index 0000000..879a747 --- /dev/null +++ b/SLS_Tests/generate_tests/token_strings.py @@ -0,0 +1,651 @@ +from typing import List, Dict, Any +from .base_tests import BaseTestGenerator, Token + + +class TokenStringTestGenerator(BaseTestGenerator): + """Generate test cases for token strings (unparsed code blocks in braces).""" + + def make_token_string_token(self, inner_tokens: List[Token]) -> Token: + """ + Create a token string token containing inner tokens. + + Args: + inner_tokens: List of tokens inside the token string + + Returns: + Token with type "token_string" and tokens array + """ + return Token(type="token_string", value=[t.__dict__ for t in inner_tokens]) + + def make_token_string_test(self, name: str, code: str, inner_tokens: List[Token]): + """ + Create a successful token string test. + + Args: + name: Test name + code: Source code + inner_tokens: Tokens inside the token string + """ + token = self.make_token_string_token(inner_tokens) + + # For operations, we push the token string as a value + op_value = {"tokens": [t.__dict__ for t in inner_tokens]} + op = self.make_push_op("token_string", op_value) + + # On the stack, it's a token_string value + stack = self.make_stack_item("token_string", op_value) + + self.add_test(name, code, [token], [op], [stack]) + + def generate_basic_tests(self): + """Generate basic token string tests.""" + # Empty token string + self.make_token_string_test( + "TokenString Empty", + "{ }", + [] + ) + + # Single token + self.make_token_string_test( + "TokenString Single Integer", + "{ 42 }", + [Token(type="i64", value=42)] + ) + + self.make_token_string_test( + "TokenString Single Identifier", + "{ dup }", + [Token(type="identifier", value="dup")] + ) + + # Two tokens + self.make_token_string_test( + "TokenString Two Integers", + "{ 2 3 }", + [ + Token(type="i64", value=2), + Token(type="i64", value=3) + ] + ) + + # Simple expression + self.make_token_string_test( + "TokenString Simple Expression", + "{ 2 3 + }", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Stack manipulation + self.make_token_string_test( + "TokenString Stack Ops", + "{ dup * }", + [ + Token(type="identifier", value="dup"), + Token(type="identifier", value="*") + ] + ) + + def generate_literal_tests(self): + """Generate tests with various literal types inside token strings.""" + # Integer literals + self.make_token_string_test( + "TokenString Integer Literals", + "{ 0 42 -10 1000 }", + [ + Token(type="i64", value=0), + Token(type="i64", value=42), + Token(type="i64", value=-10), + Token(type="i64", value=1000) + ] + ) + + # Float literals + self.make_token_string_test( + "TokenString Float Literals", + "{ 3.14 -2.5 0.0 }", + [ + Token(type="f64", value=3.14), + Token(type="f64", value=-2.5), + Token(type="f64", value=0.0) + ] + ) + + # String literals + self.make_token_string_test( + "TokenString String Literal", + '{ "hello" }', + [Token(type="String", value="hello")] + ) + + self.make_token_string_test( + "TokenString Multiple Strings", + '{ "hello" "world" }', + [ + Token(type="String", value="hello"), + Token(type="String", value="world") + ] + ) + + # Character literals + self.make_token_string_test( + "TokenString Char Literal", + "{ 'A' }", + [Token(type="char", value='A')] + ) + + # Boolean literals + self.make_token_string_test( + "TokenString Boolean Literals", + "{ true false }", + [ + Token(type="bool", value=True), + Token(type="bool", value=False) + ] + ) + + # Mixed literals + self.make_token_string_test( + "TokenString Mixed Literals", + '{ 42 3.14 "hello" true \'A\' }', + [ + Token(type="i64", value=42), + Token(type="f64", value=3.14), + Token(type="String", value="hello"), + Token(type="bool", value=True), + Token(type="char", value='A') + ] + ) + + def generate_identifier_tests(self): + """Generate tests with various identifiers inside token strings.""" + # Multiple identifiers + self.make_token_string_test( + "TokenString Multiple Identifiers", + "{ dup swap over }", + [ + Token(type="identifier", value="dup"), + Token(type="identifier", value="swap"), + Token(type="identifier", value="over") + ] + ) + + # Identifier literals (with ::) + self.make_token_string_test( + "TokenString Identifier Literals", + "{ ::x ::y }", + [ + Token(type="identifier_literal", value="x"), + Token(type="identifier_literal", value="y") + ] + ) + + # Mixed identifiers and literals + self.make_token_string_test( + "TokenString Mixed Identifiers", + "{ ::Point get x swap }", + [ + Token(type="identifier_literal", value="Point"), + Token(type="identifier", value="get"), + Token(type="identifier", value="x"), + Token(type="identifier", value="swap") + ] + ) + + def generate_nested_tests(self): + """Generate tests with nested token strings.""" + # Single nesting + inner_tokens = [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + inner_token_string = self.make_token_string_token(inner_tokens) + + self.make_token_string_test( + "TokenString Nested Single", + "{ { 2 3 + } }", + [inner_token_string] + ) + + # Nested with other tokens + self.make_token_string_test( + "TokenString Nested With Others", + "{ x { dup * } }", + [ + Token(type="identifier", value="x"), + self.make_token_string_token([ + Token(type="identifier", value="dup"), + Token(type="identifier", value="*") + ]) + ] + ) + + # Multiple nested + self.make_token_string_test( + "TokenString Multiple Nested", + "{ { 2 3 + } { 4 5 * } }", + [ + self.make_token_string_token([ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ]), + self.make_token_string_token([ + Token(type="i64", value=4), + Token(type="i64", value=5), + Token(type="identifier", value="*") + ]) + ] + ) + + # Double nesting + deepest = self.make_token_string_token([Token(type="i64", value=42)]) + middle = self.make_token_string_token([deepest]) + + self.make_token_string_test( + "TokenString Double Nested", + "{ { { 42 } } }", + [middle] + ) + + # Complex nesting with mixed content + self.make_token_string_test( + "TokenString Complex Nesting", + "{ 1 { 2 { 3 } 4 } 5 }", + [ + Token(type="i64", value=1), + self.make_token_string_token([ + Token(type="i64", value=2), + self.make_token_string_token([ + Token(type="i64", value=3) + ]), + Token(type="i64", value=4) + ]), + Token(type="i64", value=5) + ] + ) + + def generate_whitespace_tests(self): + """Generate tests with various whitespace patterns.""" + # No whitespace inside + self.make_token_string_test( + "TokenString No Whitespace", + "{2 3 +}", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Extra whitespace + self.make_token_string_test( + "TokenString Extra Whitespace", + "{ 2 3 + }", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Leading whitespace outside + self.make_token_string_test( + "TokenString Leading Whitespace Outside", + " { 2 3 + }", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Trailing whitespace outside + self.make_token_string_test( + "TokenString Trailing Whitespace Outside", + "{ 2 3 + } ", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Tabs + self.make_token_string_test( + "TokenString With Tabs", + "{\t2\t3\t+\t}", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + def generate_multiline_tests(self): + """Generate tests with multiline token strings.""" + # Simple multiline + self.make_token_string_test( + "TokenString Multiline Simple", + "{\n 2 3 +\n}", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Multiple lines with tokens + self.make_token_string_test( + "TokenString Multiline Multiple", + "{\n dup\n *\n 2\n +\n}", + [ + Token(type="identifier", value="dup"), + Token(type="identifier", value="*"), + Token(type="i64", value=2), + Token(type="identifier", value="+") + ] + ) + + # Mixed line breaks + self.make_token_string_test( + "TokenString Mixed Line Breaks", + "{ 1 2\n3 4\n\n5 6 }", + [ + Token(type="i64", value=1), + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="i64", value=4), + Token(type="i64", value=5), + Token(type="i64", value=6) + ] + ) + + # Indented multiline + self.make_token_string_test( + "TokenString Indented Multiline", + "{\n dup 0 >\n { }\n { 0 swap - }\n if\n}", + [ + Token(type="identifier", value="dup"), + Token(type="i64", value=0), + Token(type="identifier", value=">"), + self.make_token_string_token([]), + self.make_token_string_token([ + Token(type="i64", value=0), + Token(type="identifier", value="swap"), + Token(type="identifier", value="-") + ]), + Token(type="identifier", value="if") + ] + ) + + def generate_comment_tests(self): + """Generate tests with comments inside token strings.""" + # Comment at end of line inside token string + self.make_token_string_test( + "TokenString Comment End Of Line", + "{ 2 3 + // add them\n}", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Multiple comments + self.make_token_string_test( + "TokenString Multiple Comments", + "{ 2 // first\n3 // second\n+ // add\n}", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Comment on its own line + self.make_token_string_test( + "TokenString Comment Own Line", + "{\n // This is a comment\n 2 3 +\n}", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Comment at start + self.make_token_string_test( + "TokenString Comment At Start", + "{ // comment\n2 3 + }", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Multiple comment lines + self.make_token_string_test( + "TokenString Multiple Comment Lines", + "{\n // First comment\n // Second comment\n 2 3 +\n}", + [ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ] + ) + + # Comments in nested token strings + self.make_token_string_test( + "TokenString Comments Nested", + "{ { 2 3 + // inner comment\n} // outer comment\n}", + [ + self.make_token_string_token([ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ]) + ] + ) + + def generate_error_tests(self): + """Generate error test cases.""" + # Unclosed token string + self.make_error_test( + "TokenString Unclosed", + "{ 2 3 +", + "Unclosed token string: missing closing brace '}'." + ) + + # Unclosed nested + self.make_error_test( + "TokenString Unclosed Nested", + "{ { 2 3 + }", + "Unclosed token string: missing closing brace '}'." + ) + + # Extra closing brace (generates two tokens: valid token string + error) + token = self.make_token_string_token([ + Token(type="i64", value=2), + Token(type="i64", value=3), + Token(type="identifier", value="+") + ]) + error_token = self.make_error_token( + "Unexpected closing brace '}' without matching opening brace." + ) + self.add_test( + "TokenString Extra Closing Brace", + "{ 2 3 + } }", + [token, error_token] + ) + + # Only closing brace + self.make_error_test( + "TokenString Only Closing Brace", + "}", + "Unexpected closing brace '}' without matching opening brace." + ) + + # Error inside token string (invalid literal) + error_inside = self.make_token_string_token([ + Token(type="i64", value=2), + Token(type="error", value="Invalid decimal literal: unexpected 'a' in decimal integer."), + Token(type="identifier", value="+") + ]) + op_value = {"tokens": [ + {"type": "i64", "value": 2}, + {"type": "error", "value": "Invalid decimal literal: unexpected 'a' in decimal integer."}, + {"type": "identifier", "value": "+"} + ]} + op = self.make_push_op("token_string", op_value) + stack = self.make_stack_item("token_string", op_value) + + self.add_test( + "TokenString Error Inside", + "{ 2 3a + }", + [error_inside], + [op], + [stack] + ) + + # Unclosed string inside token string + error_string = self.make_token_string_token([ + Token(type="error", value="Invalid string literal: unclosed string literal.") + ]) + op_value_str = {"tokens": [ + {"type": "error", "value": "Invalid string literal: unclosed string literal."} + ]} + op_str = self.make_push_op("token_string", op_value_str) + stack_str = self.make_stack_item("token_string", op_value_str) + + self.add_test( + "TokenString Unclosed String Inside", + '{ "hello }', + [error_string], + [op_str], + [stack_str] + ) + + def generate_complex_tests(self): + """Generate complex realistic test cases.""" + # Function body + self.make_token_string_test( + "TokenString Function Body", + "{ dup * }", + [ + Token(type="identifier", value="dup"), + Token(type="identifier", value="*") + ] + ) + + # If statement branches + self.make_token_string_test( + "TokenString If Branches", + '{ "positive" print }', + [ + Token(type="String", value="positive"), + Token(type="identifier", value="print") + ] + ) + + # Loop body + self.make_token_string_test( + "TokenString Loop Body", + "{ dup print 1 + }", + [ + Token(type="identifier", value="dup"), + Token(type="identifier", value="print"), + Token(type="i64", value=1), + Token(type="identifier", value="+") + ] + ) + + # Struct definition + self.make_token_string_test( + "TokenString Struct Fields", + "{ x: y: }", + [ + Token(type="identifier", value="x"), + Token(type="identifier", value=":"), + Token(type="identifier", value="y"), + Token(type="identifier", value=":") + ] + ) + + # Lambda expression + self.make_token_string_test( + "TokenString Lambda", + "{ 2 * }", + [ + Token(type="i64", value=2), + Token(type="identifier", value="*") + ] + ) + + # Array map operation + self.make_token_string_test( + "TokenString Array Map", + "{ dup * }", + [ + Token(type="identifier", value="dup"), + Token(type="identifier", value="*") + ] + ) + + # Conditional with nested token strings + self.make_token_string_test( + "TokenString Conditional Complex", + "{\n dup 0 >\n { dup * }\n { drop 0 }\n if\n}", + [ + Token(type="identifier", value="dup"), + Token(type="i64", value=0), + Token(type="identifier", value=">"), + self.make_token_string_token([ + Token(type="identifier", value="dup"), + Token(type="identifier", value="*") + ]), + self.make_token_string_token([ + Token(type="identifier", value="drop"), + Token(type="i64", value=0) + ]), + Token(type="identifier", value="if") + ] + ) + + def generate_all_tests(self) -> List[Dict[str, Any]]: + """Generate all token string test cases.""" + # Basic tests + self.generate_basic_tests() + + # Literal types + self.generate_literal_tests() + + # Identifiers + self.generate_identifier_tests() + + # Nested token strings + self.generate_nested_tests() + + # Whitespace handling + self.generate_whitespace_tests() + + # Multiline token strings + self.generate_multiline_tests() + + # Comments inside token strings + self.generate_comment_tests() + + # Error cases + self.generate_error_tests() + + # Complex realistic cases + self.generate_complex_tests() + + return self.get_tests() +