from typing import List, Dict, Any from .base_tests import BaseTestGenerator, Token class TokenStringTestGenerator(BaseTestGenerator): """Generate test cases for token strings (unparsed code blocks in braces).""" def generate_basic_tests(self): """Generate basic token string tests.""" # Empty token string self.make_token_string_test( "TokenString Empty", "{ }", [] ) # Single token self.make_token_string_test( "TokenString Single Integer", "{ 42 }", [Token(type="i64", value=42)] ) self.make_token_string_test( "TokenString Single Identifier", "{ dup }", [Token(type="identifier", value="dup")] ) # Two tokens self.make_token_string_test( "TokenString Two Integers", "{ 2 3 }", [ Token(type="i64", value=2), Token(type="i64", value=3) ] ) # Simple expression self.make_token_string_test( "TokenString Simple Expression", "{ 2 3 + }", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Stack manipulation self.make_token_string_test( "TokenString Stack Ops", "{ dup * }", [ Token(type="identifier", value="dup"), Token(type="identifier", value="*") ] ) def generate_literal_tests(self): """Generate tests with various literal types inside token strings.""" # Integer literals self.make_token_string_test( "TokenString Integer Literals", "{ 0 42 -10 1000 }", [ Token(type="i64", value=0), Token(type="i64", value=42), Token(type="i64", value=-10), Token(type="i64", value=1000) ] ) # Float literals self.make_token_string_test( "TokenString Float Literals", "{ 3.14 -2.5 0.0 }", [ Token(type="f64", value=3.14), Token(type="f64", value=-2.5), Token(type="f64", value=0.0) ] ) # String literals if self.ENABLE_STRINGS: self.make_token_string_test( "TokenString String Literal", '{ "hello" }', [Token(type="String", value="hello")] ) self.make_token_string_test( "TokenString Multiple Strings", '{ "hello" "world" }', [ Token(type="String", value="hello"), Token(type="String", value="world") ] ) # Character literals self.make_token_string_test( "TokenString Char Literal", "{ 'A' }", [Token(type="char", value='A')] ) # Boolean literals self.make_token_string_test( "TokenString Boolean Literals", "{ true false }", [ Token(type="bool", value=True), Token(type="bool", value=False) ] ) # Mixed literals if self.ENABLE_STRINGS: self.make_token_string_test( "TokenString Mixed Literals", '{ 42 3.14 "hello" true \'A\' }', [ Token(type="i64", value=42), Token(type="f64", value=3.14), Token(type="String", value="hello"), Token(type="bool", value=True), Token(type="char", value='A') ] ) def generate_identifier_tests(self): """Generate tests with various identifiers inside token strings.""" # Multiple identifiers self.make_token_string_test( "TokenString Multiple Identifiers", "{ dup swap over }", [ Token(type="identifier", value="dup"), Token(type="identifier", value="swap"), Token(type="identifier", value="over") ] ) # Identifier literals (with ::) self.make_token_string_test( "TokenString Identifier Literals", "{ ::x ::y }", [ Token(type="identifier_literal", value="x"), Token(type="identifier_literal", value="y") ] ) # Mixed identifiers and literals self.make_token_string_test( "TokenString Mixed Identifiers", "{ ::Point get x swap }", [ Token(type="identifier_literal", value="Point"), Token(type="identifier", value="get"), Token(type="identifier", value="x"), Token(type="identifier", value="swap") ] ) def generate_nested_tests(self): """Generate tests with nested token strings.""" # Single nesting inner_tokens = [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] inner_token_string = self.make_token_string_token(inner_tokens) self.make_token_string_test( "TokenString Nested Single", "{ { 2 3 + } }", [inner_token_string] ) # Nested with other tokens self.make_token_string_test( "TokenString Nested With Others", "{ x { dup * } }", [ Token(type="identifier", value="x"), self.make_token_string_token([ Token(type="identifier", value="dup"), Token(type="identifier", value="*") ]) ] ) # Multiple nested self.make_token_string_test( "TokenString Multiple Nested", "{ { 2 3 + } { 4 5 * } }", [ self.make_token_string_token([ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ]), self.make_token_string_token([ Token(type="i64", value=4), Token(type="i64", value=5), Token(type="identifier", value="*") ]) ] ) # Double nesting deepest = self.make_token_string_token([Token(type="i64", value=42)]) middle = self.make_token_string_token([deepest]) self.make_token_string_test( "TokenString Double Nested", "{ { { 42 } } }", [middle] ) # Complex nesting with mixed content self.make_token_string_test( "TokenString Complex Nesting", "{ 1 { 2 { 3 } 4 } 5 }", [ Token(type="i64", value=1), self.make_token_string_token([ Token(type="i64", value=2), self.make_token_string_token([ Token(type="i64", value=3) ]), Token(type="i64", value=4) ]), Token(type="i64", value=5) ] ) def generate_whitespace_tests(self): """Generate tests with various whitespace patterns.""" # No whitespace inside self.make_token_string_test( "TokenString No Whitespace", "{2 3 +}", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Extra whitespace self.make_token_string_test( "TokenString Extra Whitespace", "{ 2 3 + }", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Leading whitespace outside self.make_token_string_test( "TokenString Leading Whitespace Outside", " { 2 3 + }", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Trailing whitespace outside self.make_token_string_test( "TokenString Trailing Whitespace Outside", "{ 2 3 + } ", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Tabs self.make_token_string_test( "TokenString With Tabs", "{\\t2\\t3\\t+\\t}", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) def generate_multiline_tests(self): """Generate tests with multiline token strings.""" # Simple multiline self.make_token_string_test( "TokenString Multiline Simple", "{\\n 2 3 +\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Multiple lines with tokens self.make_token_string_test( "TokenString Multiline Multiple", "{\\n dup\\n *\\n 2\\n +\\n}", [ Token(type="identifier", value="dup"), Token(type="identifier", value="*"), Token(type="i64", value=2), Token(type="identifier", value="+") ] ) # Mixed line breaks self.make_token_string_test( "TokenString Mixed Line Breaks", "{ 1 2\\n3 4\\n\\n5 6 }", [ Token(type="i64", value=1), Token(type="i64", value=2), Token(type="i64", value=3), Token(type="i64", value=4), Token(type="i64", value=5), Token(type="i64", value=6) ] ) # Indented multiline self.make_token_string_test( "TokenString Indented Multiline", "{\\n dup 0 >\\n { }\\n { 0 swap - }\\n if\\n}", [ Token(type="identifier", value="dup"), Token(type="i64", value=0), Token(type="identifier", value=">"), self.make_token_string_token([]), self.make_token_string_token([ Token(type="i64", value=0), Token(type="identifier", value="swap"), Token(type="identifier", value="-") ]), Token(type="identifier", value="if") ] ) def generate_comment_tests(self): """Generate tests with comments inside token strings.""" # Comment at end of line inside token string self.make_token_string_test( "TokenString Comment End Of Line", "{ 2 3 + // add them\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Multiple comments self.make_token_string_test( "TokenString Multiple Comments", "{ 2 // first\\n3 // second\\n+ // add\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Comment on its own line self.make_token_string_test( "TokenString Comment Own Line", "{\\n // This is a comment\\n 2 3 +\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Comment at start self.make_token_string_test( "TokenString Comment At Start", "{ // comment\\n2 3 + }", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Multiple comment lines self.make_token_string_test( "TokenString Multiple Comment Lines", "{\\n // First comment\\n // Second comment\\n 2 3 +\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ] ) # Comments in nested token strings self.make_token_string_test( "TokenString Comments Nested", "{ { 2 3 + // inner comment\\n} // outer comment\\n}", [ self.make_token_string_token([ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ]) ] ) def generate_error_tests(self): """Generate error test cases.""" # Unclosed token string self.make_error_test( "TokenString Unclosed", "{ 2 3 +", "Unclosed token string: missing closing brace '}'." ) # Unclosed nested self.make_error_test( "TokenString Unclosed Nested", "{ { 2 3 + }", "Unclosed token string: missing closing brace '}'." ) # Extra closing brace (generates two tokens: valid token string + error) token = self.make_token_string_token([ Token(type="i64", value=2), Token(type="i64", value=3), Token(type="identifier", value="+") ]) error_token = self.make_error_token( "Unexpected closing brace '}' without matching opening brace." ) self.add_test( "TokenString Extra Closing Brace", "{ 2 3 + } }", [token, error_token] ) # Only closing brace self.make_error_test( "TokenString Only Closing Brace", "}", "Unexpected closing brace '}' without matching opening brace." ) # Error inside token string (invalid literal) error_inside = self.make_token_string_token([ Token(type="i64", value=2), Token(type="error", value="Invalid decimal literal: unexpected 'a' in decimal integer."), Token(type="identifier", value="+") ]) op_value = {"tokens": [ {"type": "i64", "value": 2}, {"type": "error", "value": "Invalid decimal literal: unexpected 'a' in decimal integer."}, {"type": "identifier", "value": "+"} ]} op = self.make_push_op("token_string", op_value) stack = self.make_stack_item("token_string", op_value) self.add_test( "TokenString Error Inside", "{ 2 3a + }", [error_inside], [op], [stack] ) # Unclosed string inside token string error_string = self.make_token_string_token([ Token(type="error", value="Invalid string literal: unclosed string literal.") ]) op_value_str = {"tokens": [ {"type": "error", "value": "Invalid string literal: unclosed string literal."} ]} op_str = self.make_push_op("token_string", op_value_str) stack_str = self.make_stack_item("token_string", op_value_str) self.add_test( "TokenString Unclosed String Inside", '{ "hello }', [error_string], [op_str], [stack_str] ) def generate_complex_tests(self): """Generate complex realistic test cases.""" # Function body self.make_token_string_test( "TokenString Function Body", "{ dup * }", [ Token(type="identifier", value="dup"), Token(type="identifier", value="*") ] ) # If statement branches if self.ENABLE_STRINGS: self.make_token_string_test( "TokenString If Branches", '{ "positive" print }', [ Token(type="String", value="positive"), Token(type="identifier", value="print") ] ) # Loop body self.make_token_string_test( "TokenString Loop Body", "{ dup print 1 + }", [ Token(type="identifier", value="dup"), Token(type="identifier", value="print"), Token(type="i64", value=1), Token(type="identifier", value="+") ] ) # Struct definition self.make_token_string_test( "TokenString Struct Fields", "{ x: y: }", [ Token(type="identifier", value="x"), Token(type="identifier", value=":"), Token(type="identifier", value="y"), Token(type="identifier", value=":") ] ) # Lambda expression self.make_token_string_test( "TokenString Lambda", "{ 2 * }", [ Token(type="i64", value=2), Token(type="identifier", value="*") ] ) # Array map operation self.make_token_string_test( "TokenString Array Map", "{ dup * }", [ Token(type="identifier", value="dup"), Token(type="identifier", value="*") ] ) # Conditional with nested token strings self.make_token_string_test( "TokenString Conditional Complex", "{\\n dup 0 >\\n { dup * }\\n { drop 0 }\\n if\\n}", [ Token(type="identifier", value="dup"), Token(type="i64", value=0), Token(type="identifier", value=">"), self.make_token_string_token([ Token(type="identifier", value="dup"), Token(type="identifier", value="*") ]), self.make_token_string_token([ Token(type="identifier", value="drop"), Token(type="i64", value=0) ]), Token(type="identifier", value="if") ] ) def generate_all_tests(self) -> List[Dict[str, Any]]: """Generate all token string test cases.""" # Basic tests self.generate_basic_tests() # Literal types self.generate_literal_tests() # Identifiers self.generate_identifier_tests() # Nested token strings self.generate_nested_tests() # Whitespace handling self.generate_whitespace_tests() # Multiline token strings self.generate_multiline_tests() # Comments inside token strings self.generate_comment_tests() # Error cases self.generate_error_tests() # Complex realistic cases self.generate_complex_tests() return self.get_tests()