Added token string test generator
This commit is contained in:
parent
d2e990fe9b
commit
6d586852d4
|
|
@ -5,6 +5,7 @@ from .float_tests import FloatTestGenerator
|
|||
from .char_tests import CharTestGenerator
|
||||
# from .string_tests import StringTestGenerator
|
||||
from .idents_and_bools_tests import IdentifierTestGenerator, BooleanTestGenerator
|
||||
from .token_strings import TokenStringTestGenerator
|
||||
|
||||
__all__ = [
|
||||
"BaseTestGenerator",
|
||||
|
|
@ -15,4 +16,5 @@ __all__ = [
|
|||
# "StringTestGenerator",
|
||||
"IdentifierTestGenerator",
|
||||
"BooleanTestGenerator",
|
||||
"TokenStringTestGenerator",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,651 @@
|
|||
from typing import List, Dict, Any
|
||||
from .base_tests import BaseTestGenerator, Token
|
||||
|
||||
|
||||
class TokenStringTestGenerator(BaseTestGenerator):
|
||||
"""Generate test cases for token strings (unparsed code blocks in braces)."""
|
||||
|
||||
def make_token_string_token(self, inner_tokens: List[Token]) -> Token:
|
||||
"""
|
||||
Create a token string token containing inner tokens.
|
||||
|
||||
Args:
|
||||
inner_tokens: List of tokens inside the token string
|
||||
|
||||
Returns:
|
||||
Token with type "token_string" and tokens array
|
||||
"""
|
||||
return Token(type="token_string", value=[t.__dict__ for t in inner_tokens])
|
||||
|
||||
def make_token_string_test(self, name: str, code: str, inner_tokens: List[Token]):
|
||||
"""
|
||||
Create a successful token string test.
|
||||
|
||||
Args:
|
||||
name: Test name
|
||||
code: Source code
|
||||
inner_tokens: Tokens inside the token string
|
||||
"""
|
||||
token = self.make_token_string_token(inner_tokens)
|
||||
|
||||
# For operations, we push the token string as a value
|
||||
op_value = {"tokens": [t.__dict__ for t in inner_tokens]}
|
||||
op = self.make_push_op("token_string", op_value)
|
||||
|
||||
# On the stack, it's a token_string value
|
||||
stack = self.make_stack_item("token_string", op_value)
|
||||
|
||||
self.add_test(name, code, [token], [op], [stack])
|
||||
|
||||
def generate_basic_tests(self):
|
||||
"""Generate basic token string tests."""
|
||||
# Empty token string
|
||||
self.make_token_string_test(
|
||||
"TokenString Empty",
|
||||
"{ }",
|
||||
[]
|
||||
)
|
||||
|
||||
# Single token
|
||||
self.make_token_string_test(
|
||||
"TokenString Single Integer",
|
||||
"{ 42 }",
|
||||
[Token(type="i64", value=42)]
|
||||
)
|
||||
|
||||
self.make_token_string_test(
|
||||
"TokenString Single Identifier",
|
||||
"{ dup }",
|
||||
[Token(type="identifier", value="dup")]
|
||||
)
|
||||
|
||||
# Two tokens
|
||||
self.make_token_string_test(
|
||||
"TokenString Two Integers",
|
||||
"{ 2 3 }",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3)
|
||||
]
|
||||
)
|
||||
|
||||
# Simple expression
|
||||
self.make_token_string_test(
|
||||
"TokenString Simple Expression",
|
||||
"{ 2 3 + }",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Stack manipulation
|
||||
self.make_token_string_test(
|
||||
"TokenString Stack Ops",
|
||||
"{ dup * }",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="*")
|
||||
]
|
||||
)
|
||||
|
||||
def generate_literal_tests(self):
|
||||
"""Generate tests with various literal types inside token strings."""
|
||||
# Integer literals
|
||||
self.make_token_string_test(
|
||||
"TokenString Integer Literals",
|
||||
"{ 0 42 -10 1000 }",
|
||||
[
|
||||
Token(type="i64", value=0),
|
||||
Token(type="i64", value=42),
|
||||
Token(type="i64", value=-10),
|
||||
Token(type="i64", value=1000)
|
||||
]
|
||||
)
|
||||
|
||||
# Float literals
|
||||
self.make_token_string_test(
|
||||
"TokenString Float Literals",
|
||||
"{ 3.14 -2.5 0.0 }",
|
||||
[
|
||||
Token(type="f64", value=3.14),
|
||||
Token(type="f64", value=-2.5),
|
||||
Token(type="f64", value=0.0)
|
||||
]
|
||||
)
|
||||
|
||||
# String literals
|
||||
self.make_token_string_test(
|
||||
"TokenString String Literal",
|
||||
'{ "hello" }',
|
||||
[Token(type="String", value="hello")]
|
||||
)
|
||||
|
||||
self.make_token_string_test(
|
||||
"TokenString Multiple Strings",
|
||||
'{ "hello" "world" }',
|
||||
[
|
||||
Token(type="String", value="hello"),
|
||||
Token(type="String", value="world")
|
||||
]
|
||||
)
|
||||
|
||||
# Character literals
|
||||
self.make_token_string_test(
|
||||
"TokenString Char Literal",
|
||||
"{ 'A' }",
|
||||
[Token(type="char", value='A')]
|
||||
)
|
||||
|
||||
# Boolean literals
|
||||
self.make_token_string_test(
|
||||
"TokenString Boolean Literals",
|
||||
"{ true false }",
|
||||
[
|
||||
Token(type="bool", value=True),
|
||||
Token(type="bool", value=False)
|
||||
]
|
||||
)
|
||||
|
||||
# Mixed literals
|
||||
self.make_token_string_test(
|
||||
"TokenString Mixed Literals",
|
||||
'{ 42 3.14 "hello" true \'A\' }',
|
||||
[
|
||||
Token(type="i64", value=42),
|
||||
Token(type="f64", value=3.14),
|
||||
Token(type="String", value="hello"),
|
||||
Token(type="bool", value=True),
|
||||
Token(type="char", value='A')
|
||||
]
|
||||
)
|
||||
|
||||
def generate_identifier_tests(self):
|
||||
"""Generate tests with various identifiers inside token strings."""
|
||||
# Multiple identifiers
|
||||
self.make_token_string_test(
|
||||
"TokenString Multiple Identifiers",
|
||||
"{ dup swap over }",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="swap"),
|
||||
Token(type="identifier", value="over")
|
||||
]
|
||||
)
|
||||
|
||||
# Identifier literals (with ::)
|
||||
self.make_token_string_test(
|
||||
"TokenString Identifier Literals",
|
||||
"{ ::x ::y }",
|
||||
[
|
||||
Token(type="identifier_literal", value="x"),
|
||||
Token(type="identifier_literal", value="y")
|
||||
]
|
||||
)
|
||||
|
||||
# Mixed identifiers and literals
|
||||
self.make_token_string_test(
|
||||
"TokenString Mixed Identifiers",
|
||||
"{ ::Point get x swap }",
|
||||
[
|
||||
Token(type="identifier_literal", value="Point"),
|
||||
Token(type="identifier", value="get"),
|
||||
Token(type="identifier", value="x"),
|
||||
Token(type="identifier", value="swap")
|
||||
]
|
||||
)
|
||||
|
||||
def generate_nested_tests(self):
|
||||
"""Generate tests with nested token strings."""
|
||||
# Single nesting
|
||||
inner_tokens = [
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
inner_token_string = self.make_token_string_token(inner_tokens)
|
||||
|
||||
self.make_token_string_test(
|
||||
"TokenString Nested Single",
|
||||
"{ { 2 3 + } }",
|
||||
[inner_token_string]
|
||||
)
|
||||
|
||||
# Nested with other tokens
|
||||
self.make_token_string_test(
|
||||
"TokenString Nested With Others",
|
||||
"{ x { dup * } }",
|
||||
[
|
||||
Token(type="identifier", value="x"),
|
||||
self.make_token_string_token([
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="*")
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
# Multiple nested
|
||||
self.make_token_string_test(
|
||||
"TokenString Multiple Nested",
|
||||
"{ { 2 3 + } { 4 5 * } }",
|
||||
[
|
||||
self.make_token_string_token([
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]),
|
||||
self.make_token_string_token([
|
||||
Token(type="i64", value=4),
|
||||
Token(type="i64", value=5),
|
||||
Token(type="identifier", value="*")
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
# Double nesting
|
||||
deepest = self.make_token_string_token([Token(type="i64", value=42)])
|
||||
middle = self.make_token_string_token([deepest])
|
||||
|
||||
self.make_token_string_test(
|
||||
"TokenString Double Nested",
|
||||
"{ { { 42 } } }",
|
||||
[middle]
|
||||
)
|
||||
|
||||
# Complex nesting with mixed content
|
||||
self.make_token_string_test(
|
||||
"TokenString Complex Nesting",
|
||||
"{ 1 { 2 { 3 } 4 } 5 }",
|
||||
[
|
||||
Token(type="i64", value=1),
|
||||
self.make_token_string_token([
|
||||
Token(type="i64", value=2),
|
||||
self.make_token_string_token([
|
||||
Token(type="i64", value=3)
|
||||
]),
|
||||
Token(type="i64", value=4)
|
||||
]),
|
||||
Token(type="i64", value=5)
|
||||
]
|
||||
)
|
||||
|
||||
def generate_whitespace_tests(self):
|
||||
"""Generate tests with various whitespace patterns."""
|
||||
# No whitespace inside
|
||||
self.make_token_string_test(
|
||||
"TokenString No Whitespace",
|
||||
"{2 3 +}",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Extra whitespace
|
||||
self.make_token_string_test(
|
||||
"TokenString Extra Whitespace",
|
||||
"{ 2 3 + }",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Leading whitespace outside
|
||||
self.make_token_string_test(
|
||||
"TokenString Leading Whitespace Outside",
|
||||
" { 2 3 + }",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Trailing whitespace outside
|
||||
self.make_token_string_test(
|
||||
"TokenString Trailing Whitespace Outside",
|
||||
"{ 2 3 + } ",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Tabs
|
||||
self.make_token_string_test(
|
||||
"TokenString With Tabs",
|
||||
"{\t2\t3\t+\t}",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
def generate_multiline_tests(self):
|
||||
"""Generate tests with multiline token strings."""
|
||||
# Simple multiline
|
||||
self.make_token_string_test(
|
||||
"TokenString Multiline Simple",
|
||||
"{\n 2 3 +\n}",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Multiple lines with tokens
|
||||
self.make_token_string_test(
|
||||
"TokenString Multiline Multiple",
|
||||
"{\n dup\n *\n 2\n +\n}",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="*"),
|
||||
Token(type="i64", value=2),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Mixed line breaks
|
||||
self.make_token_string_test(
|
||||
"TokenString Mixed Line Breaks",
|
||||
"{ 1 2\n3 4\n\n5 6 }",
|
||||
[
|
||||
Token(type="i64", value=1),
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="i64", value=4),
|
||||
Token(type="i64", value=5),
|
||||
Token(type="i64", value=6)
|
||||
]
|
||||
)
|
||||
|
||||
# Indented multiline
|
||||
self.make_token_string_test(
|
||||
"TokenString Indented Multiline",
|
||||
"{\n dup 0 >\n { }\n { 0 swap - }\n if\n}",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="i64", value=0),
|
||||
Token(type="identifier", value=">"),
|
||||
self.make_token_string_token([]),
|
||||
self.make_token_string_token([
|
||||
Token(type="i64", value=0),
|
||||
Token(type="identifier", value="swap"),
|
||||
Token(type="identifier", value="-")
|
||||
]),
|
||||
Token(type="identifier", value="if")
|
||||
]
|
||||
)
|
||||
|
||||
def generate_comment_tests(self):
|
||||
"""Generate tests with comments inside token strings."""
|
||||
# Comment at end of line inside token string
|
||||
self.make_token_string_test(
|
||||
"TokenString Comment End Of Line",
|
||||
"{ 2 3 + // add them\n}",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Multiple comments
|
||||
self.make_token_string_test(
|
||||
"TokenString Multiple Comments",
|
||||
"{ 2 // first\n3 // second\n+ // add\n}",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Comment on its own line
|
||||
self.make_token_string_test(
|
||||
"TokenString Comment Own Line",
|
||||
"{\n // This is a comment\n 2 3 +\n}",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Comment at start
|
||||
self.make_token_string_test(
|
||||
"TokenString Comment At Start",
|
||||
"{ // comment\n2 3 + }",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Multiple comment lines
|
||||
self.make_token_string_test(
|
||||
"TokenString Multiple Comment Lines",
|
||||
"{\n // First comment\n // Second comment\n 2 3 +\n}",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Comments in nested token strings
|
||||
self.make_token_string_test(
|
||||
"TokenString Comments Nested",
|
||||
"{ { 2 3 + // inner comment\n} // outer comment\n}",
|
||||
[
|
||||
self.make_token_string_token([
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
def generate_error_tests(self):
|
||||
"""Generate error test cases."""
|
||||
# Unclosed token string
|
||||
self.make_error_test(
|
||||
"TokenString Unclosed",
|
||||
"{ 2 3 +",
|
||||
"Unclosed token string: missing closing brace '}'."
|
||||
)
|
||||
|
||||
# Unclosed nested
|
||||
self.make_error_test(
|
||||
"TokenString Unclosed Nested",
|
||||
"{ { 2 3 + }",
|
||||
"Unclosed token string: missing closing brace '}'."
|
||||
)
|
||||
|
||||
# Extra closing brace (generates two tokens: valid token string + error)
|
||||
token = self.make_token_string_token([
|
||||
Token(type="i64", value=2),
|
||||
Token(type="i64", value=3),
|
||||
Token(type="identifier", value="+")
|
||||
])
|
||||
error_token = self.make_error_token(
|
||||
"Unexpected closing brace '}' without matching opening brace."
|
||||
)
|
||||
self.add_test(
|
||||
"TokenString Extra Closing Brace",
|
||||
"{ 2 3 + } }",
|
||||
[token, error_token]
|
||||
)
|
||||
|
||||
# Only closing brace
|
||||
self.make_error_test(
|
||||
"TokenString Only Closing Brace",
|
||||
"}",
|
||||
"Unexpected closing brace '}' without matching opening brace."
|
||||
)
|
||||
|
||||
# Error inside token string (invalid literal)
|
||||
error_inside = self.make_token_string_token([
|
||||
Token(type="i64", value=2),
|
||||
Token(type="error", value="Invalid decimal literal: unexpected 'a' in decimal integer."),
|
||||
Token(type="identifier", value="+")
|
||||
])
|
||||
op_value = {"tokens": [
|
||||
{"type": "i64", "value": 2},
|
||||
{"type": "error", "value": "Invalid decimal literal: unexpected 'a' in decimal integer."},
|
||||
{"type": "identifier", "value": "+"}
|
||||
]}
|
||||
op = self.make_push_op("token_string", op_value)
|
||||
stack = self.make_stack_item("token_string", op_value)
|
||||
|
||||
self.add_test(
|
||||
"TokenString Error Inside",
|
||||
"{ 2 3a + }",
|
||||
[error_inside],
|
||||
[op],
|
||||
[stack]
|
||||
)
|
||||
|
||||
# Unclosed string inside token string
|
||||
error_string = self.make_token_string_token([
|
||||
Token(type="error", value="Invalid string literal: unclosed string literal.")
|
||||
])
|
||||
op_value_str = {"tokens": [
|
||||
{"type": "error", "value": "Invalid string literal: unclosed string literal."}
|
||||
]}
|
||||
op_str = self.make_push_op("token_string", op_value_str)
|
||||
stack_str = self.make_stack_item("token_string", op_value_str)
|
||||
|
||||
self.add_test(
|
||||
"TokenString Unclosed String Inside",
|
||||
'{ "hello }',
|
||||
[error_string],
|
||||
[op_str],
|
||||
[stack_str]
|
||||
)
|
||||
|
||||
def generate_complex_tests(self):
|
||||
"""Generate complex realistic test cases."""
|
||||
# Function body
|
||||
self.make_token_string_test(
|
||||
"TokenString Function Body",
|
||||
"{ dup * }",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="*")
|
||||
]
|
||||
)
|
||||
|
||||
# If statement branches
|
||||
self.make_token_string_test(
|
||||
"TokenString If Branches",
|
||||
'{ "positive" print }',
|
||||
[
|
||||
Token(type="String", value="positive"),
|
||||
Token(type="identifier", value="print")
|
||||
]
|
||||
)
|
||||
|
||||
# Loop body
|
||||
self.make_token_string_test(
|
||||
"TokenString Loop Body",
|
||||
"{ dup print 1 + }",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="print"),
|
||||
Token(type="i64", value=1),
|
||||
Token(type="identifier", value="+")
|
||||
]
|
||||
)
|
||||
|
||||
# Struct definition
|
||||
self.make_token_string_test(
|
||||
"TokenString Struct Fields",
|
||||
"{ x: y: }",
|
||||
[
|
||||
Token(type="identifier", value="x"),
|
||||
Token(type="identifier", value=":"),
|
||||
Token(type="identifier", value="y"),
|
||||
Token(type="identifier", value=":")
|
||||
]
|
||||
)
|
||||
|
||||
# Lambda expression
|
||||
self.make_token_string_test(
|
||||
"TokenString Lambda",
|
||||
"{ 2 * }",
|
||||
[
|
||||
Token(type="i64", value=2),
|
||||
Token(type="identifier", value="*")
|
||||
]
|
||||
)
|
||||
|
||||
# Array map operation
|
||||
self.make_token_string_test(
|
||||
"TokenString Array Map",
|
||||
"{ dup * }",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="*")
|
||||
]
|
||||
)
|
||||
|
||||
# Conditional with nested token strings
|
||||
self.make_token_string_test(
|
||||
"TokenString Conditional Complex",
|
||||
"{\n dup 0 >\n { dup * }\n { drop 0 }\n if\n}",
|
||||
[
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="i64", value=0),
|
||||
Token(type="identifier", value=">"),
|
||||
self.make_token_string_token([
|
||||
Token(type="identifier", value="dup"),
|
||||
Token(type="identifier", value="*")
|
||||
]),
|
||||
self.make_token_string_token([
|
||||
Token(type="identifier", value="drop"),
|
||||
Token(type="i64", value=0)
|
||||
]),
|
||||
Token(type="identifier", value="if")
|
||||
]
|
||||
)
|
||||
|
||||
def generate_all_tests(self) -> List[Dict[str, Any]]:
|
||||
"""Generate all token string test cases."""
|
||||
# Basic tests
|
||||
self.generate_basic_tests()
|
||||
|
||||
# Literal types
|
||||
self.generate_literal_tests()
|
||||
|
||||
# Identifiers
|
||||
self.generate_identifier_tests()
|
||||
|
||||
# Nested token strings
|
||||
self.generate_nested_tests()
|
||||
|
||||
# Whitespace handling
|
||||
self.generate_whitespace_tests()
|
||||
|
||||
# Multiline token strings
|
||||
self.generate_multiline_tests()
|
||||
|
||||
# Comments inside token strings
|
||||
self.generate_comment_tests()
|
||||
|
||||
# Error cases
|
||||
self.generate_error_tests()
|
||||
|
||||
# Complex realistic cases
|
||||
self.generate_complex_tests()
|
||||
|
||||
return self.get_tests()
|
||||
|
||||
Loading…
Reference in New Issue