652 lines
21 KiB
Python
652 lines
21 KiB
Python
from typing import List, Dict, Any
|
|
from .base_tests import BaseTestGenerator, Token
|
|
|
|
|
|
class TokenStringTestGenerator(BaseTestGenerator):
|
|
"""Generate test cases for token strings (unparsed code blocks in braces)."""
|
|
|
|
def make_token_string_token(self, inner_tokens: List[Token]) -> Token:
|
|
"""
|
|
Create a token string token containing inner tokens.
|
|
|
|
Args:
|
|
inner_tokens: List of tokens inside the token string
|
|
|
|
Returns:
|
|
Token with type "token_string" and tokens array
|
|
"""
|
|
return Token(type="token_string", value=[t.__dict__ for t in inner_tokens])
|
|
|
|
def make_token_string_test(self, name: str, code: str, inner_tokens: List[Token]):
|
|
"""
|
|
Create a successful token string test.
|
|
|
|
Args:
|
|
name: Test name
|
|
code: Source code
|
|
inner_tokens: Tokens inside the token string
|
|
"""
|
|
token = self.make_token_string_token(inner_tokens)
|
|
|
|
# For operations, we push the token string as a value
|
|
op_value = {"tokens": [t.__dict__ for t in inner_tokens]}
|
|
op = self.make_push_op("token_string", op_value)
|
|
|
|
# On the stack, it's a token_string value
|
|
stack = self.make_stack_item("token_string", op_value)
|
|
|
|
self.add_test(name, code, [token], [op], [stack])
|
|
|
|
def generate_basic_tests(self):
|
|
"""Generate basic token string tests."""
|
|
# Empty token string
|
|
self.make_token_string_test(
|
|
"TokenString Empty",
|
|
"{ }",
|
|
[]
|
|
)
|
|
|
|
# Single token
|
|
self.make_token_string_test(
|
|
"TokenString Single Integer",
|
|
"{ 42 }",
|
|
[Token(type="i64", value=42)]
|
|
)
|
|
|
|
self.make_token_string_test(
|
|
"TokenString Single Identifier",
|
|
"{ dup }",
|
|
[Token(type="identifier", value="dup")]
|
|
)
|
|
|
|
# Two tokens
|
|
self.make_token_string_test(
|
|
"TokenString Two Integers",
|
|
"{ 2 3 }",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3)
|
|
]
|
|
)
|
|
|
|
# Simple expression
|
|
self.make_token_string_test(
|
|
"TokenString Simple Expression",
|
|
"{ 2 3 + }",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Stack manipulation
|
|
self.make_token_string_test(
|
|
"TokenString Stack Ops",
|
|
"{ dup * }",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="*")
|
|
]
|
|
)
|
|
|
|
def generate_literal_tests(self):
|
|
"""Generate tests with various literal types inside token strings."""
|
|
# Integer literals
|
|
self.make_token_string_test(
|
|
"TokenString Integer Literals",
|
|
"{ 0 42 -10 1000 }",
|
|
[
|
|
Token(type="i64", value=0),
|
|
Token(type="i64", value=42),
|
|
Token(type="i64", value=-10),
|
|
Token(type="i64", value=1000)
|
|
]
|
|
)
|
|
|
|
# Float literals
|
|
self.make_token_string_test(
|
|
"TokenString Float Literals",
|
|
"{ 3.14 -2.5 0.0 }",
|
|
[
|
|
Token(type="f64", value=3.14),
|
|
Token(type="f64", value=-2.5),
|
|
Token(type="f64", value=0.0)
|
|
]
|
|
)
|
|
|
|
# String literals
|
|
self.make_token_string_test(
|
|
"TokenString String Literal",
|
|
'{ "hello" }',
|
|
[Token(type="String", value="hello")]
|
|
)
|
|
|
|
self.make_token_string_test(
|
|
"TokenString Multiple Strings",
|
|
'{ "hello" "world" }',
|
|
[
|
|
Token(type="String", value="hello"),
|
|
Token(type="String", value="world")
|
|
]
|
|
)
|
|
|
|
# Character literals
|
|
self.make_token_string_test(
|
|
"TokenString Char Literal",
|
|
"{ 'A' }",
|
|
[Token(type="char", value='A')]
|
|
)
|
|
|
|
# Boolean literals
|
|
self.make_token_string_test(
|
|
"TokenString Boolean Literals",
|
|
"{ true false }",
|
|
[
|
|
Token(type="bool", value=True),
|
|
Token(type="bool", value=False)
|
|
]
|
|
)
|
|
|
|
# Mixed literals
|
|
self.make_token_string_test(
|
|
"TokenString Mixed Literals",
|
|
'{ 42 3.14 "hello" true \'A\' }',
|
|
[
|
|
Token(type="i64", value=42),
|
|
Token(type="f64", value=3.14),
|
|
Token(type="String", value="hello"),
|
|
Token(type="bool", value=True),
|
|
Token(type="char", value='A')
|
|
]
|
|
)
|
|
|
|
def generate_identifier_tests(self):
|
|
"""Generate tests with various identifiers inside token strings."""
|
|
# Multiple identifiers
|
|
self.make_token_string_test(
|
|
"TokenString Multiple Identifiers",
|
|
"{ dup swap over }",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="swap"),
|
|
Token(type="identifier", value="over")
|
|
]
|
|
)
|
|
|
|
# Identifier literals (with ::)
|
|
self.make_token_string_test(
|
|
"TokenString Identifier Literals",
|
|
"{ ::x ::y }",
|
|
[
|
|
Token(type="identifier_literal", value="x"),
|
|
Token(type="identifier_literal", value="y")
|
|
]
|
|
)
|
|
|
|
# Mixed identifiers and literals
|
|
self.make_token_string_test(
|
|
"TokenString Mixed Identifiers",
|
|
"{ ::Point get x swap }",
|
|
[
|
|
Token(type="identifier_literal", value="Point"),
|
|
Token(type="identifier", value="get"),
|
|
Token(type="identifier", value="x"),
|
|
Token(type="identifier", value="swap")
|
|
]
|
|
)
|
|
|
|
def generate_nested_tests(self):
|
|
"""Generate tests with nested token strings."""
|
|
# Single nesting
|
|
inner_tokens = [
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
inner_token_string = self.make_token_string_token(inner_tokens)
|
|
|
|
self.make_token_string_test(
|
|
"TokenString Nested Single",
|
|
"{ { 2 3 + } }",
|
|
[inner_token_string]
|
|
)
|
|
|
|
# Nested with other tokens
|
|
self.make_token_string_test(
|
|
"TokenString Nested With Others",
|
|
"{ x { dup * } }",
|
|
[
|
|
Token(type="identifier", value="x"),
|
|
self.make_token_string_token([
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="*")
|
|
])
|
|
]
|
|
)
|
|
|
|
# Multiple nested
|
|
self.make_token_string_test(
|
|
"TokenString Multiple Nested",
|
|
"{ { 2 3 + } { 4 5 * } }",
|
|
[
|
|
self.make_token_string_token([
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]),
|
|
self.make_token_string_token([
|
|
Token(type="i64", value=4),
|
|
Token(type="i64", value=5),
|
|
Token(type="identifier", value="*")
|
|
])
|
|
]
|
|
)
|
|
|
|
# Double nesting
|
|
deepest = self.make_token_string_token([Token(type="i64", value=42)])
|
|
middle = self.make_token_string_token([deepest])
|
|
|
|
self.make_token_string_test(
|
|
"TokenString Double Nested",
|
|
"{ { { 42 } } }",
|
|
[middle]
|
|
)
|
|
|
|
# Complex nesting with mixed content
|
|
self.make_token_string_test(
|
|
"TokenString Complex Nesting",
|
|
"{ 1 { 2 { 3 } 4 } 5 }",
|
|
[
|
|
Token(type="i64", value=1),
|
|
self.make_token_string_token([
|
|
Token(type="i64", value=2),
|
|
self.make_token_string_token([
|
|
Token(type="i64", value=3)
|
|
]),
|
|
Token(type="i64", value=4)
|
|
]),
|
|
Token(type="i64", value=5)
|
|
]
|
|
)
|
|
|
|
def generate_whitespace_tests(self):
|
|
"""Generate tests with various whitespace patterns."""
|
|
# No whitespace inside
|
|
self.make_token_string_test(
|
|
"TokenString No Whitespace",
|
|
"{2 3 +}",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Extra whitespace
|
|
self.make_token_string_test(
|
|
"TokenString Extra Whitespace",
|
|
"{ 2 3 + }",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Leading whitespace outside
|
|
self.make_token_string_test(
|
|
"TokenString Leading Whitespace Outside",
|
|
" { 2 3 + }",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Trailing whitespace outside
|
|
self.make_token_string_test(
|
|
"TokenString Trailing Whitespace Outside",
|
|
"{ 2 3 + } ",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Tabs
|
|
self.make_token_string_test(
|
|
"TokenString With Tabs",
|
|
"{\t2\t3\t+\t}",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
def generate_multiline_tests(self):
|
|
"""Generate tests with multiline token strings."""
|
|
# Simple multiline
|
|
self.make_token_string_test(
|
|
"TokenString Multiline Simple",
|
|
"{\n 2 3 +\n}",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Multiple lines with tokens
|
|
self.make_token_string_test(
|
|
"TokenString Multiline Multiple",
|
|
"{\n dup\n *\n 2\n +\n}",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="*"),
|
|
Token(type="i64", value=2),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Mixed line breaks
|
|
self.make_token_string_test(
|
|
"TokenString Mixed Line Breaks",
|
|
"{ 1 2\n3 4\n\n5 6 }",
|
|
[
|
|
Token(type="i64", value=1),
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="i64", value=4),
|
|
Token(type="i64", value=5),
|
|
Token(type="i64", value=6)
|
|
]
|
|
)
|
|
|
|
# Indented multiline
|
|
self.make_token_string_test(
|
|
"TokenString Indented Multiline",
|
|
"{\n dup 0 >\n { }\n { 0 swap - }\n if\n}",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="i64", value=0),
|
|
Token(type="identifier", value=">"),
|
|
self.make_token_string_token([]),
|
|
self.make_token_string_token([
|
|
Token(type="i64", value=0),
|
|
Token(type="identifier", value="swap"),
|
|
Token(type="identifier", value="-")
|
|
]),
|
|
Token(type="identifier", value="if")
|
|
]
|
|
)
|
|
|
|
def generate_comment_tests(self):
|
|
"""Generate tests with comments inside token strings."""
|
|
# Comment at end of line inside token string
|
|
self.make_token_string_test(
|
|
"TokenString Comment End Of Line",
|
|
"{ 2 3 + // add them\n}",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Multiple comments
|
|
self.make_token_string_test(
|
|
"TokenString Multiple Comments",
|
|
"{ 2 // first\n3 // second\n+ // add\n}",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Comment on its own line
|
|
self.make_token_string_test(
|
|
"TokenString Comment Own Line",
|
|
"{\n // This is a comment\n 2 3 +\n}",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Comment at start
|
|
self.make_token_string_test(
|
|
"TokenString Comment At Start",
|
|
"{ // comment\n2 3 + }",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Multiple comment lines
|
|
self.make_token_string_test(
|
|
"TokenString Multiple Comment Lines",
|
|
"{\n // First comment\n // Second comment\n 2 3 +\n}",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Comments in nested token strings
|
|
self.make_token_string_test(
|
|
"TokenString Comments Nested",
|
|
"{ { 2 3 + // inner comment\n} // outer comment\n}",
|
|
[
|
|
self.make_token_string_token([
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
])
|
|
]
|
|
)
|
|
|
|
def generate_error_tests(self):
|
|
"""Generate error test cases."""
|
|
# Unclosed token string
|
|
self.make_error_test(
|
|
"TokenString Unclosed",
|
|
"{ 2 3 +",
|
|
"Unclosed token string: missing closing brace '}'."
|
|
)
|
|
|
|
# Unclosed nested
|
|
self.make_error_test(
|
|
"TokenString Unclosed Nested",
|
|
"{ { 2 3 + }",
|
|
"Unclosed token string: missing closing brace '}'."
|
|
)
|
|
|
|
# Extra closing brace (generates two tokens: valid token string + error)
|
|
token = self.make_token_string_token([
|
|
Token(type="i64", value=2),
|
|
Token(type="i64", value=3),
|
|
Token(type="identifier", value="+")
|
|
])
|
|
error_token = self.make_error_token(
|
|
"Unexpected closing brace '}' without matching opening brace."
|
|
)
|
|
self.add_test(
|
|
"TokenString Extra Closing Brace",
|
|
"{ 2 3 + } }",
|
|
[token, error_token]
|
|
)
|
|
|
|
# Only closing brace
|
|
self.make_error_test(
|
|
"TokenString Only Closing Brace",
|
|
"}",
|
|
"Unexpected closing brace '}' without matching opening brace."
|
|
)
|
|
|
|
# Error inside token string (invalid literal)
|
|
error_inside = self.make_token_string_token([
|
|
Token(type="i64", value=2),
|
|
Token(type="error", value="Invalid decimal literal: unexpected 'a' in decimal integer."),
|
|
Token(type="identifier", value="+")
|
|
])
|
|
op_value = {"tokens": [
|
|
{"type": "i64", "value": 2},
|
|
{"type": "error", "value": "Invalid decimal literal: unexpected 'a' in decimal integer."},
|
|
{"type": "identifier", "value": "+"}
|
|
]}
|
|
op = self.make_push_op("token_string", op_value)
|
|
stack = self.make_stack_item("token_string", op_value)
|
|
|
|
self.add_test(
|
|
"TokenString Error Inside",
|
|
"{ 2 3a + }",
|
|
[error_inside],
|
|
[op],
|
|
[stack]
|
|
)
|
|
|
|
# Unclosed string inside token string
|
|
error_string = self.make_token_string_token([
|
|
Token(type="error", value="Invalid string literal: unclosed string literal.")
|
|
])
|
|
op_value_str = {"tokens": [
|
|
{"type": "error", "value": "Invalid string literal: unclosed string literal."}
|
|
]}
|
|
op_str = self.make_push_op("token_string", op_value_str)
|
|
stack_str = self.make_stack_item("token_string", op_value_str)
|
|
|
|
self.add_test(
|
|
"TokenString Unclosed String Inside",
|
|
'{ "hello }',
|
|
[error_string],
|
|
[op_str],
|
|
[stack_str]
|
|
)
|
|
|
|
def generate_complex_tests(self):
|
|
"""Generate complex realistic test cases."""
|
|
# Function body
|
|
self.make_token_string_test(
|
|
"TokenString Function Body",
|
|
"{ dup * }",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="*")
|
|
]
|
|
)
|
|
|
|
# If statement branches
|
|
self.make_token_string_test(
|
|
"TokenString If Branches",
|
|
'{ "positive" print }',
|
|
[
|
|
Token(type="String", value="positive"),
|
|
Token(type="identifier", value="print")
|
|
]
|
|
)
|
|
|
|
# Loop body
|
|
self.make_token_string_test(
|
|
"TokenString Loop Body",
|
|
"{ dup print 1 + }",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="print"),
|
|
Token(type="i64", value=1),
|
|
Token(type="identifier", value="+")
|
|
]
|
|
)
|
|
|
|
# Struct definition
|
|
self.make_token_string_test(
|
|
"TokenString Struct Fields",
|
|
"{ x: y: }",
|
|
[
|
|
Token(type="identifier", value="x"),
|
|
Token(type="identifier", value=":"),
|
|
Token(type="identifier", value="y"),
|
|
Token(type="identifier", value=":")
|
|
]
|
|
)
|
|
|
|
# Lambda expression
|
|
self.make_token_string_test(
|
|
"TokenString Lambda",
|
|
"{ 2 * }",
|
|
[
|
|
Token(type="i64", value=2),
|
|
Token(type="identifier", value="*")
|
|
]
|
|
)
|
|
|
|
# Array map operation
|
|
self.make_token_string_test(
|
|
"TokenString Array Map",
|
|
"{ dup * }",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="*")
|
|
]
|
|
)
|
|
|
|
# Conditional with nested token strings
|
|
self.make_token_string_test(
|
|
"TokenString Conditional Complex",
|
|
"{\n dup 0 >\n { dup * }\n { drop 0 }\n if\n}",
|
|
[
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="i64", value=0),
|
|
Token(type="identifier", value=">"),
|
|
self.make_token_string_token([
|
|
Token(type="identifier", value="dup"),
|
|
Token(type="identifier", value="*")
|
|
]),
|
|
self.make_token_string_token([
|
|
Token(type="identifier", value="drop"),
|
|
Token(type="i64", value=0)
|
|
]),
|
|
Token(type="identifier", value="if")
|
|
]
|
|
)
|
|
|
|
def generate_all_tests(self) -> List[Dict[str, Any]]:
|
|
"""Generate all token string test cases."""
|
|
# Basic tests
|
|
self.generate_basic_tests()
|
|
|
|
# Literal types
|
|
self.generate_literal_tests()
|
|
|
|
# Identifiers
|
|
self.generate_identifier_tests()
|
|
|
|
# Nested token strings
|
|
self.generate_nested_tests()
|
|
|
|
# Whitespace handling
|
|
self.generate_whitespace_tests()
|
|
|
|
# Multiline token strings
|
|
self.generate_multiline_tests()
|
|
|
|
# Comments inside token strings
|
|
self.generate_comment_tests()
|
|
|
|
# Error cases
|
|
self.generate_error_tests()
|
|
|
|
# Complex realistic cases
|
|
self.generate_complex_tests()
|
|
|
|
return self.get_tests()
|
|
|