From ad26c41463aba30336e82ccf649307b5f5b49ee4 Mon Sep 17 00:00:00 2001 From: Kyler Date: Sun, 23 Nov 2025 21:07:25 -0700 Subject: [PATCH] Got token_string tests converting to C --- SLS_C/tests/lexer_tests.c | 562 +++++++- SLS_Tests/cases.yaml | 1526 ++++++++++++++++++++- SLS_Tests/generate_tests/__init__.py | 4 +- SLS_Tests/generate_tests/base_tests.py | 37 + SLS_Tests/generate_tests/string_tests.py | 4 + SLS_Tests/generate_tests/token_strings.py | 125 +- SLS_Tests/yaml_to_c_tests.py | 153 +++ 7 files changed, 2249 insertions(+), 162 deletions(-) diff --git a/SLS_C/tests/lexer_tests.c b/SLS_C/tests/lexer_tests.c index db4bd3f..decf5b4 100644 --- a/SLS_C/tests/lexer_tests.c +++ b/SLS_C/tests/lexer_tests.c @@ -16,7 +16,7 @@ #include "tests/tests.h" -static const size_t NUM_OF_TESTS = 344; +static const size_t NUM_OF_TESTS = 389; static TestResult test_Empty_Statement() { LexerTest test = start_up_test(SLS_STR("Empty_Statement"), SLS_STR("")); @@ -2073,6 +2073,16 @@ static TestResult test_Char_Right_Brace() { return pass_test(&test, result); } +static TestResult test_Char_Escape_Tab() { + LexerTest test = start_up_test(SLS_STR("Char Escape Tab"), SLS_STR("'\\t'")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_character_value(&test, result, i++, &(uint8_t){9})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + static TestResult test_Char_Escape_Null_character() { LexerTest test = start_up_test(SLS_STR("Char Escape Null character"), SLS_STR("'\\0'")); LexerResult result = lexical_analysis(&test.lexer_info); @@ -2083,16 +2093,6 @@ static TestResult test_Char_Escape_Null_character() { return pass_test(&test, result); } -static TestResult test_Char_Escape_Backslash() { - LexerTest test = start_up_test(SLS_STR("Char Escape Backslash"), SLS_STR("'\\\\'")); - LexerResult result = lexical_analysis(&test.lexer_info); - if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); - size_t i = 0; - if (test_character_value(&test, result, i++, &(uint8_t){92})) return test.result; - if (test_eof_value(&test, result, i++, 0)) return test.result; - return pass_test(&test, result); -} - static TestResult test_Char_Escape_Carriage_return() { LexerTest test = start_up_test(SLS_STR("Char Escape Carriage return"), SLS_STR("'\\r'")); LexerResult result = lexical_analysis(&test.lexer_info); @@ -2103,26 +2103,6 @@ static TestResult test_Char_Escape_Carriage_return() { return pass_test(&test, result); } -static TestResult test_Char_Escape_Single_quote() { - LexerTest test = start_up_test(SLS_STR("Char Escape Single quote"), SLS_STR("'\\''")); - LexerResult result = lexical_analysis(&test.lexer_info); - if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); - size_t i = 0; - if (test_character_value(&test, result, i++, &(uint8_t){39})) return test.result; - if (test_eof_value(&test, result, i++, 0)) return test.result; - return pass_test(&test, result); -} - -static TestResult test_Char_Escape_Tab() { - LexerTest test = start_up_test(SLS_STR("Char Escape Tab"), SLS_STR("'\\t'")); - LexerResult result = lexical_analysis(&test.lexer_info); - if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); - size_t i = 0; - if (test_character_value(&test, result, i++, &(uint8_t){9})) return test.result; - if (test_eof_value(&test, result, i++, 0)) return test.result; - return pass_test(&test, result); -} - static TestResult test_Char_Escape_Newline() { LexerTest test = start_up_test(SLS_STR("Char Escape Newline"), SLS_STR("'\\n'")); LexerResult result = lexical_analysis(&test.lexer_info); @@ -2133,6 +2113,26 @@ static TestResult test_Char_Escape_Newline() { return pass_test(&test, result); } +static TestResult test_Char_Escape_Backslash() { + LexerTest test = start_up_test(SLS_STR("Char Escape Backslash"), SLS_STR("'\\\\'")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_character_value(&test, result, i++, &(uint8_t){92})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_Char_Escape_Single_quote() { + LexerTest test = start_up_test(SLS_STR("Char Escape Single quote"), SLS_STR("'\\''")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_character_value(&test, result, i++, &(uint8_t){39})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + static TestResult test_Char_Hex_Lowercase_A() { LexerTest test = start_up_test(SLS_STR("Char Hex Lowercase A"), SLS_STR("'\\x61'")); LexerResult result = lexical_analysis(&test.lexer_info); @@ -3425,6 +3425,453 @@ static TestResult test_Bool_Three_Values() { return pass_test(&test, result); } +static TestResult test_TokenString_Empty() { + LexerTest test = start_up_test(SLS_STR("TokenString Empty"), SLS_STR("{ }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){0, NULL})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Single_Integer() { + LexerTest test = start_up_test(SLS_STR("TokenString Single Integer"), SLS_STR("{ 42 }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 42}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Single_Identifier() { + LexerTest test = start_up_test(SLS_STR("TokenString Single Identifier"), SLS_STR("{ dup }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Two_Integers() { + LexerTest test = start_up_test(SLS_STR("TokenString Two Integers"), SLS_STR("{ 2 3 }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Simple_Expression() { + LexerTest test = start_up_test(SLS_STR("TokenString Simple Expression"), SLS_STR("{ 2 3 + }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Stack_Ops() { + LexerTest test = start_up_test(SLS_STR("TokenString Stack Ops"), SLS_STR("{ dup * }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Integer_Literals() { + LexerTest test = start_up_test(SLS_STR("TokenString Integer Literals"), SLS_STR("{ 0 42 -10 1000 }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){4, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 0}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 42}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, -10}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 1000}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Float_Literals() { + LexerTest test = start_up_test(SLS_STR("TokenString Float Literals"), SLS_STR("{ 3.14 -2.5 0.0 }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_double_value, .value = &(double){3.14}}, {.token_handler = test_double_value, .value = &(double){-2.5}}, {.token_handler = test_double_value, .value = &(double){0.0}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Char_Literal() { + LexerTest test = start_up_test(SLS_STR("TokenString Char Literal"), SLS_STR("{ 'A' }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_character_value, .value = &(uint8_t){65}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Boolean_Literals() { + LexerTest test = start_up_test(SLS_STR("TokenString Boolean Literals"), SLS_STR("{ true false }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_boolean_value, .value = &(Boolean){TRUE}}, {.token_handler = test_boolean_value, .value = &(Boolean){FALSE}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Multiple_Identifiers() { + LexerTest test = start_up_test(SLS_STR("TokenString Multiple Identifiers"), SLS_STR("{ dup swap over }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("swap")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("over")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Identifier_Literals() { + LexerTest test = start_up_test(SLS_STR("TokenString Identifier Literals"), SLS_STR("{ ::x ::y }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){TRUE, SLS_STR("x")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){TRUE, SLS_STR("y")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Mixed_Identifiers() { + LexerTest test = start_up_test(SLS_STR("TokenString Mixed Identifiers"), SLS_STR("{ ::Point get x swap }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){4, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){TRUE, SLS_STR("Point")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("get")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("x")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("swap")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Nested_Single() { + LexerTest test = start_up_test(SLS_STR("TokenString Nested Single"), SLS_STR("{ { 2 3 + } }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_token_string_value, .value = &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Nested_With_Others() { + LexerTest test = start_up_test(SLS_STR("TokenString Nested With Others"), SLS_STR("{ x { dup * } }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("x")}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}}}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Multiple_Nested() { + LexerTest test = start_up_test(SLS_STR("TokenString Multiple Nested"), SLS_STR("{ { 2 3 + } { 4 5 * } }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_token_string_value, .value = &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 4}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 5}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}}}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Double_Nested() { + LexerTest test = start_up_test(SLS_STR("TokenString Double Nested"), SLS_STR("{ { { 42 } } }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_token_string_value, .value = &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_token_string_value, .value = &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 42}}}}}}}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Complex_Nesting() { + LexerTest test = start_up_test(SLS_STR("TokenString Complex Nesting"), SLS_STR("{ 1 { 2 { 3 } 4 } 5 }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 1}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}}}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 4}}}}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 5}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_No_Whitespace() { + LexerTest test = start_up_test(SLS_STR("TokenString No Whitespace"), SLS_STR("{2 3 +}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Extra_Whitespace() { + LexerTest test = start_up_test(SLS_STR("TokenString Extra Whitespace"), SLS_STR("{ 2 3 + }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Leading_Whitespace_Outside() { + LexerTest test = start_up_test(SLS_STR("TokenString Leading Whitespace Outside"), SLS_STR(" { 2 3 + }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Trailing_Whitespace_Outside() { + LexerTest test = start_up_test(SLS_STR("TokenString Trailing Whitespace Outside"), SLS_STR("{ 2 3 + } ")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_With_Tabs() { + LexerTest test = start_up_test(SLS_STR("TokenString With Tabs"), SLS_STR("{\t2\t3\t+\t}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Multiline_Simple() { + LexerTest test = start_up_test(SLS_STR("TokenString Multiline Simple"), SLS_STR("{\n 2 3 +\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Multiline_Multiple() { + LexerTest test = start_up_test(SLS_STR("TokenString Multiline Multiple"), SLS_STR("{\n dup\n *\n 2\n +\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){4, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Mixed_Line_Breaks() { + LexerTest test = start_up_test(SLS_STR("TokenString Mixed Line Breaks"), SLS_STR("{ 1 2\n3 4\n\n5 6 }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){6, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 1}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 4}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 5}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 6}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Indented_Multiline() { + LexerTest test = start_up_test(SLS_STR("TokenString Indented Multiline"), SLS_STR("{\n dup 0 >\n { }\n { 0 swap - }\n if\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){6, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 0}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR(">")}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){0, NULL}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 0}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("swap")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("-")}}}}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("if")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Comment_End_Of_Line() { + LexerTest test = start_up_test(SLS_STR("TokenString Comment End Of Line"), SLS_STR("{ 2 3 + // add them\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Multiple_Comments() { + LexerTest test = start_up_test(SLS_STR("TokenString Multiple Comments"), SLS_STR("{ 2 // first\n3 // second\n+ // add\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Comment_Own_Line() { + LexerTest test = start_up_test(SLS_STR("TokenString Comment Own Line"), SLS_STR("{\n // This is a comment\n 2 3 +\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Comment_At_Start() { + LexerTest test = start_up_test(SLS_STR("TokenString Comment At Start"), SLS_STR("{ // comment\n2 3 + }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Multiple_Comment_Lines() { + LexerTest test = start_up_test(SLS_STR("TokenString Multiple Comment Lines"), SLS_STR("{\n // First comment\n // Second comment\n 2 3 +\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Comments_Nested() { + LexerTest test = start_up_test(SLS_STR("TokenString Comments Nested"), SLS_STR("{ { 2 3 + // inner comment\n} // outer comment\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_token_string_value, .value = &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Unclosed() { + LexerTest test = start_up_test(SLS_STR("TokenString Unclosed"), SLS_STR("{ 2 3 +")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_for_error(&test, result, i++, SLS_STR("Unclosed token string: missing closing brace '}'."))) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Unclosed_Nested() { + LexerTest test = start_up_test(SLS_STR("TokenString Unclosed Nested"), SLS_STR("{ { 2 3 + }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_for_error(&test, result, i++, SLS_STR("Unclosed token string: missing closing brace '}'."))) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Extra_Closing_Brace() { + LexerTest test = start_up_test(SLS_STR("TokenString Extra Closing Brace"), SLS_STR("{ 2 3 + } }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 3}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_for_error(&test, result, i++, SLS_STR("Unexpected closing brace '}' without matching opening brace."))) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Only_Closing_Brace() { + LexerTest test = start_up_test(SLS_STR("TokenString Only Closing Brace"), SLS_STR("}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_for_error(&test, result, i++, SLS_STR("Unexpected closing brace '}' without matching opening brace."))) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Error_Inside() { + LexerTest test = start_up_test(SLS_STR("TokenString Error Inside"), SLS_STR("{ 2 3a + }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){3, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_for_error, .value = SLS_STR("Invalid decimal literal: unexpected 'a' in decimal integer.")}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Unclosed_String_Inside() { + LexerTest test = start_up_test(SLS_STR("TokenString Unclosed String Inside"), SLS_STR("{ \"hello }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){1, (TestTokenValue[]){{.token_handler = test_for_error, .value = SLS_STR("Invalid string literal: unclosed string literal.")}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Function_Body() { + LexerTest test = start_up_test(SLS_STR("TokenString Function Body"), SLS_STR("{ dup * }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Loop_Body() { + LexerTest test = start_up_test(SLS_STR("TokenString Loop Body"), SLS_STR("{ dup print 1 + }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){4, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("print")}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 1}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("+")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Struct_Fields() { + LexerTest test = start_up_test(SLS_STR("TokenString Struct Fields"), SLS_STR("{ x: y: }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){4, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("x")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR(":")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("y")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR(":")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Lambda() { + LexerTest test = start_up_test(SLS_STR("TokenString Lambda"), SLS_STR("{ 2 * }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 2}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Array_Map() { + LexerTest test = start_up_test(SLS_STR("TokenString Array Map"), SLS_STR("{ dup * }")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + +static TestResult test_TokenString_Conditional_Complex() { + LexerTest test = start_up_test(SLS_STR("TokenString Conditional Complex"), SLS_STR("{\n dup 0 >\n { dup * }\n { drop 0 }\n if\n}")); + LexerResult result = lexical_analysis(&test.lexer_info); + if (result.type == SLS_ERROR) return error_fail_test(&test, result, result.error); + size_t i = 0; + if (test_token_string_value(&test, result, i, &(TestTokenStringValue){6, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 0}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR(">")}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("dup")}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("*")}}}}}, {.token_handler = test_token_string_value, .value = &(TestTokenStringValue){2, (TestTokenValue[]){{.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("drop")}}, {.token_handler = test_integer_value, .value = &(TestIntegerValue){INTEGER_I64, 0}}}}}, {.token_handler = test_identifier_value, .value = &(TestIdentifierValue){FALSE, SLS_STR("if")}}}})) return test.result; + if (test_eof_value(&test, result, i++, 0)) return test.result; + return pass_test(&test, result); +} + TestsReport run_lexer_tests() { TestsReport test_report = (TestsReport) { .section = SLS_STR("lexer_tests"), @@ -3642,12 +4089,12 @@ TestsReport run_lexer_tests() { test_report.tests[i++] = test_Char_Right_Bracket(); test_report.tests[i++] = test_Char_Left_Brace(); test_report.tests[i++] = test_Char_Right_Brace(); - test_report.tests[i++] = test_Char_Escape_Null_character(); - test_report.tests[i++] = test_Char_Escape_Backslash(); - test_report.tests[i++] = test_Char_Escape_Carriage_return(); - test_report.tests[i++] = test_Char_Escape_Single_quote(); test_report.tests[i++] = test_Char_Escape_Tab(); + test_report.tests[i++] = test_Char_Escape_Null_character(); + test_report.tests[i++] = test_Char_Escape_Carriage_return(); test_report.tests[i++] = test_Char_Escape_Newline(); + test_report.tests[i++] = test_Char_Escape_Backslash(); + test_report.tests[i++] = test_Char_Escape_Single_quote(); test_report.tests[i++] = test_Char_Hex_Lowercase_A(); test_report.tests[i++] = test_Char_Hex_Uppercase_A(); test_report.tests[i++] = test_Char_Hex_Space(); @@ -3778,6 +4225,51 @@ TestsReport run_lexer_tests() { test_report.tests[i++] = test_Bool_Multiple_True_False(); test_report.tests[i++] = test_Bool_Multiple_Same(); test_report.tests[i++] = test_Bool_Three_Values(); + test_report.tests[i++] = test_TokenString_Empty(); + test_report.tests[i++] = test_TokenString_Single_Integer(); + test_report.tests[i++] = test_TokenString_Single_Identifier(); + test_report.tests[i++] = test_TokenString_Two_Integers(); + test_report.tests[i++] = test_TokenString_Simple_Expression(); + test_report.tests[i++] = test_TokenString_Stack_Ops(); + test_report.tests[i++] = test_TokenString_Integer_Literals(); + test_report.tests[i++] = test_TokenString_Float_Literals(); + test_report.tests[i++] = test_TokenString_Char_Literal(); + test_report.tests[i++] = test_TokenString_Boolean_Literals(); + test_report.tests[i++] = test_TokenString_Multiple_Identifiers(); + test_report.tests[i++] = test_TokenString_Identifier_Literals(); + test_report.tests[i++] = test_TokenString_Mixed_Identifiers(); + test_report.tests[i++] = test_TokenString_Nested_Single(); + test_report.tests[i++] = test_TokenString_Nested_With_Others(); + test_report.tests[i++] = test_TokenString_Multiple_Nested(); + test_report.tests[i++] = test_TokenString_Double_Nested(); + test_report.tests[i++] = test_TokenString_Complex_Nesting(); + test_report.tests[i++] = test_TokenString_No_Whitespace(); + test_report.tests[i++] = test_TokenString_Extra_Whitespace(); + test_report.tests[i++] = test_TokenString_Leading_Whitespace_Outside(); + test_report.tests[i++] = test_TokenString_Trailing_Whitespace_Outside(); + test_report.tests[i++] = test_TokenString_With_Tabs(); + test_report.tests[i++] = test_TokenString_Multiline_Simple(); + test_report.tests[i++] = test_TokenString_Multiline_Multiple(); + test_report.tests[i++] = test_TokenString_Mixed_Line_Breaks(); + test_report.tests[i++] = test_TokenString_Indented_Multiline(); + test_report.tests[i++] = test_TokenString_Comment_End_Of_Line(); + test_report.tests[i++] = test_TokenString_Multiple_Comments(); + test_report.tests[i++] = test_TokenString_Comment_Own_Line(); + test_report.tests[i++] = test_TokenString_Comment_At_Start(); + test_report.tests[i++] = test_TokenString_Multiple_Comment_Lines(); + test_report.tests[i++] = test_TokenString_Comments_Nested(); + test_report.tests[i++] = test_TokenString_Unclosed(); + test_report.tests[i++] = test_TokenString_Unclosed_Nested(); + test_report.tests[i++] = test_TokenString_Extra_Closing_Brace(); + test_report.tests[i++] = test_TokenString_Only_Closing_Brace(); + test_report.tests[i++] = test_TokenString_Error_Inside(); + test_report.tests[i++] = test_TokenString_Unclosed_String_Inside(); + test_report.tests[i++] = test_TokenString_Function_Body(); + test_report.tests[i++] = test_TokenString_Loop_Body(); + test_report.tests[i++] = test_TokenString_Struct_Fields(); + test_report.tests[i++] = test_TokenString_Lambda(); + test_report.tests[i++] = test_TokenString_Array_Map(); + test_report.tests[i++] = test_TokenString_Conditional_Complex(); return test_report; } diff --git a/SLS_Tests/cases.yaml b/SLS_Tests/cases.yaml index 0b24ed5..10ca9bb 100644 --- a/SLS_Tests/cases.yaml +++ b/SLS_Tests/cases.yaml @@ -2317,54 +2317,6 @@ stack_final: - type: char value: '}' -- name: Char Escape Null character - code: '''\\0''' - tokens: - - type: char - value: "\0" - operations: - - function: push - type: char - value: "\0" - stack_final: - - type: char - value: "\0" -- name: Char Escape Backslash - code: '''\\\\''' - tokens: - - type: char - value: \ - operations: - - function: push - type: char - value: \ - stack_final: - - type: char - value: \ -- name: Char Escape Carriage return - code: '''\\r''' - tokens: - - type: char - value: "\r" - operations: - - function: push - type: char - value: "\r" - stack_final: - - type: char - value: "\r" -- name: Char Escape Single quote - code: '''\\''''' - tokens: - - type: char - value: '''' - operations: - - function: push - type: char - value: '''' - stack_final: - - type: char - value: '''' - name: Char Escape Tab code: '''\\t''' tokens: @@ -2377,6 +2329,30 @@ stack_final: - type: char value: "\t" +- name: Char Escape Null character + code: '''\\0''' + tokens: + - type: char + value: "\0" + operations: + - function: push + type: char + value: "\0" + stack_final: + - type: char + value: "\0" +- name: Char Escape Carriage return + code: '''\\r''' + tokens: + - type: char + value: "\r" + operations: + - function: push + type: char + value: "\r" + stack_final: + - type: char + value: "\r" - name: Char Escape Newline code: '''\\n''' tokens: @@ -2395,6 +2371,30 @@ value: ' ' +- name: Char Escape Backslash + code: '''\\\\''' + tokens: + - type: char + value: \ + operations: + - function: push + type: char + value: \ + stack_final: + - type: char + value: \ +- name: Char Escape Single quote + code: '''\\''''' + tokens: + - type: char + value: '''' + operations: + - function: push + type: char + value: '''' + stack_final: + - type: char + value: '''' - name: Char Hex Lowercase A code: '''\\x61''' tokens: @@ -3905,3 +3905,1433 @@ value: false - type: bool value: true +- name: TokenString Empty + code: '{ }' + tokens: + - type: token_string + value: [] + operations: + - function: push + type: token_string + value: + tokens: [] + stack_final: + - type: token_string + value: + tokens: [] +- name: TokenString Single Integer + code: '{ 42 }' + tokens: + - type: token_string + value: + - type: i64 + value: 42 + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 42 + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 42 +- name: TokenString Single Identifier + code: '{ dup }' + tokens: + - type: token_string + value: + - type: identifier + value: dup + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup +- name: TokenString Two Integers + code: '{ 2 3 }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 +- name: TokenString Simple Expression + code: '{ 2 3 + }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Stack Ops + code: '{ dup * }' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' +- name: TokenString Integer Literals + code: '{ 0 42 -10 1000 }' + tokens: + - type: token_string + value: + - type: i64 + value: 0 + - type: i64 + value: 42 + - type: i64 + value: -10 + - type: i64 + value: 1000 + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 0 + - type: i64 + value: 42 + - type: i64 + value: -10 + - type: i64 + value: 1000 + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 0 + - type: i64 + value: 42 + - type: i64 + value: -10 + - type: i64 + value: 1000 +- name: TokenString Float Literals + code: '{ 3.14 -2.5 0.0 }' + tokens: + - type: token_string + value: + - type: f64 + value: 3.14 + - type: f64 + value: -2.5 + - type: f64 + value: 0.0 + operations: + - function: push + type: token_string + value: + tokens: + - type: f64 + value: 3.14 + - type: f64 + value: -2.5 + - type: f64 + value: 0.0 + stack_final: + - type: token_string + value: + tokens: + - type: f64 + value: 3.14 + - type: f64 + value: -2.5 + - type: f64 + value: 0.0 +- name: TokenString Char Literal + code: '{ ''A'' }' + tokens: + - type: token_string + value: + - type: char + value: A + operations: + - function: push + type: token_string + value: + tokens: + - type: char + value: A + stack_final: + - type: token_string + value: + tokens: + - type: char + value: A +- name: TokenString Boolean Literals + code: '{ true false }' + tokens: + - type: token_string + value: + - type: bool + value: true + - type: bool + value: false + operations: + - function: push + type: token_string + value: + tokens: + - type: bool + value: true + - type: bool + value: false + stack_final: + - type: token_string + value: + tokens: + - type: bool + value: true + - type: bool + value: false +- name: TokenString Multiple Identifiers + code: '{ dup swap over }' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: swap + - type: identifier + value: over + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: swap + - type: identifier + value: over + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: swap + - type: identifier + value: over +- name: TokenString Identifier Literals + code: '{ ::x ::y }' + tokens: + - type: token_string + value: + - type: identifier_literal + value: x + - type: identifier_literal + value: y + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier_literal + value: x + - type: identifier_literal + value: y + stack_final: + - type: token_string + value: + tokens: + - type: identifier_literal + value: x + - type: identifier_literal + value: y +- name: TokenString Mixed Identifiers + code: '{ ::Point get x swap }' + tokens: + - type: token_string + value: + - type: identifier_literal + value: Point + - type: identifier + value: get + - type: identifier + value: x + - type: identifier + value: swap + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier_literal + value: Point + - type: identifier + value: get + - type: identifier + value: x + - type: identifier + value: swap + stack_final: + - type: token_string + value: + tokens: + - type: identifier_literal + value: Point + - type: identifier + value: get + - type: identifier + value: x + - type: identifier + value: swap +- name: TokenString Nested Single + code: '{ { 2 3 + } }' + tokens: + - type: token_string + value: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Nested With Others + code: '{ x { dup * } }' + tokens: + - type: token_string + value: + - type: identifier + value: x + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: x + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: x + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' +- name: TokenString Multiple Nested + code: '{ { 2 3 + } { 4 5 * } }' + tokens: + - type: token_string + value: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + - type: token_string + value: + - type: i64 + value: 4 + - type: i64 + value: 5 + - type: identifier + value: '*' + operations: + - function: push + type: token_string + value: + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + - type: token_string + value: + - type: i64 + value: 4 + - type: i64 + value: 5 + - type: identifier + value: '*' + stack_final: + - type: token_string + value: + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + - type: token_string + value: + - type: i64 + value: 4 + - type: i64 + value: 5 + - type: identifier + value: '*' +- name: TokenString Double Nested + code: '{ { { 42 } } }' + tokens: + - type: token_string + value: + - type: token_string + value: + - type: token_string + value: + - type: i64 + value: 42 + operations: + - function: push + type: token_string + value: + tokens: + - type: token_string + value: + - type: token_string + value: + - type: i64 + value: 42 + stack_final: + - type: token_string + value: + tokens: + - type: token_string + value: + - type: token_string + value: + - type: i64 + value: 42 +- name: TokenString Complex Nesting + code: '{ 1 { 2 { 3 } 4 } 5 }' + tokens: + - type: token_string + value: + - type: i64 + value: 1 + - type: token_string + value: + - type: i64 + value: 2 + - type: token_string + value: + - type: i64 + value: 3 + - type: i64 + value: 4 + - type: i64 + value: 5 + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 1 + - type: token_string + value: + - type: i64 + value: 2 + - type: token_string + value: + - type: i64 + value: 3 + - type: i64 + value: 4 + - type: i64 + value: 5 + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 1 + - type: token_string + value: + - type: i64 + value: 2 + - type: token_string + value: + - type: i64 + value: 3 + - type: i64 + value: 4 + - type: i64 + value: 5 +- name: TokenString No Whitespace + code: '{2 3 +}' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Extra Whitespace + code: '{ 2 3 + }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Leading Whitespace Outside + code: ' { 2 3 + }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Trailing Whitespace Outside + code: '{ 2 3 + } ' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString With Tabs + code: '{\t2\t3\t+\t}' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Multiline Simple + code: '{\n 2 3 +\n}' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Multiline Multiple + code: '{\n dup\n *\n 2\n +\n}' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + - type: i64 + value: 2 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' + - type: i64 + value: 2 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' + - type: i64 + value: 2 + - type: identifier + value: + +- name: TokenString Mixed Line Breaks + code: '{ 1 2\n3 4\n\n5 6 }' + tokens: + - type: token_string + value: + - type: i64 + value: 1 + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: i64 + value: 4 + - type: i64 + value: 5 + - type: i64 + value: 6 + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 1 + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: i64 + value: 4 + - type: i64 + value: 5 + - type: i64 + value: 6 + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 1 + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: i64 + value: 4 + - type: i64 + value: 5 + - type: i64 + value: 6 +- name: TokenString Indented Multiline + code: '{\n dup 0 >\n { }\n { 0 swap - }\n if\n}' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: i64 + value: 0 + - type: identifier + value: '>' + - type: token_string + value: [] + - type: token_string + value: + - type: i64 + value: 0 + - type: identifier + value: swap + - type: identifier + value: '-' + - type: identifier + value: if + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: i64 + value: 0 + - type: identifier + value: '>' + - type: token_string + value: [] + - type: token_string + value: + - type: i64 + value: 0 + - type: identifier + value: swap + - type: identifier + value: '-' + - type: identifier + value: if + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: i64 + value: 0 + - type: identifier + value: '>' + - type: token_string + value: [] + - type: token_string + value: + - type: i64 + value: 0 + - type: identifier + value: swap + - type: identifier + value: '-' + - type: identifier + value: if +- name: TokenString Comment End Of Line + code: '{ 2 3 + // add them\n}' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Multiple Comments + code: '{ 2 // first\n3 // second\n+ // add\n}' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Comment Own Line + code: '{\n // This is a comment\n 2 3 +\n}' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Comment At Start + code: '{ // comment\n2 3 + }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Multiple Comment Lines + code: '{\n // First comment\n // Second comment\n 2 3 +\n}' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Comments Nested + code: '{ { 2 3 + // inner comment\n} // outer comment\n}' + tokens: + - type: token_string + value: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + +- name: TokenString Unclosed + code: '{ 2 3 +' + tokens: + - type: error + value: 'Unclosed token string: missing closing brace ''}''.' +- name: TokenString Unclosed Nested + code: '{ { 2 3 + }' + tokens: + - type: error + value: 'Unclosed token string: missing closing brace ''}''.' +- name: TokenString Extra Closing Brace + code: '{ 2 3 + } }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: i64 + value: 3 + - type: identifier + value: + + - type: error + value: Unexpected closing brace '}' without matching opening brace. +- name: TokenString Only Closing Brace + code: '}' + tokens: + - type: error + value: Unexpected closing brace '}' without matching opening brace. +- name: TokenString Error Inside + code: '{ 2 3a + }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: error + value: 'Invalid decimal literal: unexpected ''a'' in decimal integer.' + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: error + value: 'Invalid decimal literal: unexpected ''a'' in decimal integer.' + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: error + value: 'Invalid decimal literal: unexpected ''a'' in decimal integer.' + - type: identifier + value: + +- name: TokenString Unclosed String Inside + code: '{ "hello }' + tokens: + - type: token_string + value: + - type: error + value: 'Invalid string literal: unclosed string literal.' + operations: + - function: push + type: token_string + value: + tokens: + - type: error + value: 'Invalid string literal: unclosed string literal.' + stack_final: + - type: token_string + value: + tokens: + - type: error + value: 'Invalid string literal: unclosed string literal.' +- name: TokenString Function Body + code: '{ dup * }' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' +- name: TokenString Loop Body + code: '{ dup print 1 + }' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: print + - type: i64 + value: 1 + - type: identifier + value: + + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: print + - type: i64 + value: 1 + - type: identifier + value: + + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: print + - type: i64 + value: 1 + - type: identifier + value: + +- name: TokenString Struct Fields + code: '{ x: y: }' + tokens: + - type: token_string + value: + - type: identifier + value: x + - type: identifier + value: ':' + - type: identifier + value: y + - type: identifier + value: ':' + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: x + - type: identifier + value: ':' + - type: identifier + value: y + - type: identifier + value: ':' + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: x + - type: identifier + value: ':' + - type: identifier + value: y + - type: identifier + value: ':' +- name: TokenString Lambda + code: '{ 2 * }' + tokens: + - type: token_string + value: + - type: i64 + value: 2 + - type: identifier + value: '*' + operations: + - function: push + type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: identifier + value: '*' + stack_final: + - type: token_string + value: + tokens: + - type: i64 + value: 2 + - type: identifier + value: '*' +- name: TokenString Array Map + code: '{ dup * }' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: identifier + value: '*' +- name: TokenString Conditional Complex + code: '{\n dup 0 >\n { dup * }\n { drop 0 }\n if\n}' + tokens: + - type: token_string + value: + - type: identifier + value: dup + - type: i64 + value: 0 + - type: identifier + value: '>' + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + - type: token_string + value: + - type: identifier + value: drop + - type: i64 + value: 0 + - type: identifier + value: if + operations: + - function: push + type: token_string + value: + tokens: + - type: identifier + value: dup + - type: i64 + value: 0 + - type: identifier + value: '>' + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + - type: token_string + value: + - type: identifier + value: drop + - type: i64 + value: 0 + - type: identifier + value: if + stack_final: + - type: token_string + value: + tokens: + - type: identifier + value: dup + - type: i64 + value: 0 + - type: identifier + value: '>' + - type: token_string + value: + - type: identifier + value: dup + - type: identifier + value: '*' + - type: token_string + value: + - type: identifier + value: drop + - type: i64 + value: 0 + - type: identifier + value: if diff --git a/SLS_Tests/generate_tests/__init__.py b/SLS_Tests/generate_tests/__init__.py index 0ed2997..68047d9 100644 --- a/SLS_Tests/generate_tests/__init__.py +++ b/SLS_Tests/generate_tests/__init__.py @@ -3,7 +3,7 @@ from .general_tests import GeneralTestGenerator from .integer_tests import IntegerTestGenerator from .float_tests import FloatTestGenerator from .char_tests import CharTestGenerator -# from .string_tests import StringTestGenerator +from .string_tests import StringTestGenerator from .idents_and_bools_tests import IdentifierTestGenerator, BooleanTestGenerator from .token_strings import TokenStringTestGenerator @@ -13,7 +13,7 @@ __all__ = [ "IntegerTestGenerator", "FloatTestGenerator", "CharTestGenerator", - # "StringTestGenerator", + "StringTestGenerator", "IdentifierTestGenerator", "BooleanTestGenerator", "TokenStringTestGenerator", diff --git a/SLS_Tests/generate_tests/base_tests.py b/SLS_Tests/generate_tests/base_tests.py index 43a876e..d4e60d6 100644 --- a/SLS_Tests/generate_tests/base_tests.py +++ b/SLS_Tests/generate_tests/base_tests.py @@ -54,6 +54,7 @@ class BaseTestGenerator(ABC): ENABLE_UNICODE = False ENABLE_EXPONENTIAL_LITERALS = False + ENABLE_STRINGS = False __generators: "ClassVar[List[Type[BaseTestGenerator]]]" = [] @@ -365,3 +366,39 @@ class BaseTestGenerator(ABC): if not self.validate_test_names_unique(): duplicates = self.get_duplicate_test_names() print(f" WARNING: Duplicate test names found: {duplicates}") + + # ========================================================================= + # Token String Helpers + # ========================================================================= + + def make_token_string_token(self, inner_tokens: List[Token]) -> Token: + """ + Create a token string token containing inner tokens. + + Args: + inner_tokens: List of tokens inside the token string + + Returns: + Token with type "token_string" and tokens array + """ + return Token(type="token_string", value=[t.__dict__ for t in inner_tokens]) + + def make_token_string_test(self, name: str, code: str, inner_tokens: List[Token]): + """ + Create a successful token string test. + + Args: + name: Test name + code: Source code + inner_tokens: Tokens inside the token string + """ + token = self.make_token_string_token(inner_tokens) + + # For operations, we push the token string as a value + op_value = {"tokens": [t.__dict__ for t in inner_tokens]} + op = self.make_push_op("token_string", op_value) + + # On the stack, it's a token_string value + stack = self.make_stack_item("token_string", op_value) + + self.add_test(name, code, [token], [op], [stack]) diff --git a/SLS_Tests/generate_tests/string_tests.py b/SLS_Tests/generate_tests/string_tests.py index 0ece35e..0ead0c6 100644 --- a/SLS_Tests/generate_tests/string_tests.py +++ b/SLS_Tests/generate_tests/string_tests.py @@ -436,6 +436,10 @@ class StringTestGenerator(BaseTestGenerator): def generate_all_tests(self) -> List[Dict[str, Any]]: """Generate all string literal test cases.""" + + if not self.ENABLE_STRINGS: + return [] + # Basic tests self.generate_basic_tests() diff --git a/SLS_Tests/generate_tests/token_strings.py b/SLS_Tests/generate_tests/token_strings.py index 879a747..3376741 100644 --- a/SLS_Tests/generate_tests/token_strings.py +++ b/SLS_Tests/generate_tests/token_strings.py @@ -5,38 +5,6 @@ from .base_tests import BaseTestGenerator, Token class TokenStringTestGenerator(BaseTestGenerator): """Generate test cases for token strings (unparsed code blocks in braces).""" - def make_token_string_token(self, inner_tokens: List[Token]) -> Token: - """ - Create a token string token containing inner tokens. - - Args: - inner_tokens: List of tokens inside the token string - - Returns: - Token with type "token_string" and tokens array - """ - return Token(type="token_string", value=[t.__dict__ for t in inner_tokens]) - - def make_token_string_test(self, name: str, code: str, inner_tokens: List[Token]): - """ - Create a successful token string test. - - Args: - name: Test name - code: Source code - inner_tokens: Tokens inside the token string - """ - token = self.make_token_string_token(inner_tokens) - - # For operations, we push the token string as a value - op_value = {"tokens": [t.__dict__ for t in inner_tokens]} - op = self.make_push_op("token_string", op_value) - - # On the stack, it's a token_string value - stack = self.make_stack_item("token_string", op_value) - - self.add_test(name, code, [token], [op], [stack]) - def generate_basic_tests(self): """Generate basic token string tests.""" # Empty token string @@ -116,20 +84,21 @@ class TokenStringTestGenerator(BaseTestGenerator): ) # String literals - self.make_token_string_test( - "TokenString String Literal", - '{ "hello" }', - [Token(type="String", value="hello")] - ) - - self.make_token_string_test( - "TokenString Multiple Strings", - '{ "hello" "world" }', - [ - Token(type="String", value="hello"), - Token(type="String", value="world") - ] - ) + if self.ENABLE_STRINGS: + self.make_token_string_test( + "TokenString String Literal", + '{ "hello" }', + [Token(type="String", value="hello")] + ) + + self.make_token_string_test( + "TokenString Multiple Strings", + '{ "hello" "world" }', + [ + Token(type="String", value="hello"), + Token(type="String", value="world") + ] + ) # Character literals self.make_token_string_test( @@ -149,17 +118,18 @@ class TokenStringTestGenerator(BaseTestGenerator): ) # Mixed literals - self.make_token_string_test( - "TokenString Mixed Literals", - '{ 42 3.14 "hello" true \'A\' }', - [ - Token(type="i64", value=42), - Token(type="f64", value=3.14), - Token(type="String", value="hello"), - Token(type="bool", value=True), - Token(type="char", value='A') - ] - ) + if self.ENABLE_STRINGS: + self.make_token_string_test( + "TokenString Mixed Literals", + '{ 42 3.14 "hello" true \'A\' }', + [ + Token(type="i64", value=42), + Token(type="f64", value=3.14), + Token(type="String", value="hello"), + Token(type="bool", value=True), + Token(type="char", value='A') + ] + ) def generate_identifier_tests(self): """Generate tests with various identifiers inside token strings.""" @@ -319,7 +289,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Tabs self.make_token_string_test( "TokenString With Tabs", - "{\t2\t3\t+\t}", + "{\\t2\\t3\\t+\\t}", [ Token(type="i64", value=2), Token(type="i64", value=3), @@ -332,7 +302,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Simple multiline self.make_token_string_test( "TokenString Multiline Simple", - "{\n 2 3 +\n}", + "{\\n 2 3 +\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), @@ -343,7 +313,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Multiple lines with tokens self.make_token_string_test( "TokenString Multiline Multiple", - "{\n dup\n *\n 2\n +\n}", + "{\\n dup\\n *\\n 2\\n +\\n}", [ Token(type="identifier", value="dup"), Token(type="identifier", value="*"), @@ -355,7 +325,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Mixed line breaks self.make_token_string_test( "TokenString Mixed Line Breaks", - "{ 1 2\n3 4\n\n5 6 }", + "{ 1 2\\n3 4\\n\\n5 6 }", [ Token(type="i64", value=1), Token(type="i64", value=2), @@ -369,7 +339,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Indented multiline self.make_token_string_test( "TokenString Indented Multiline", - "{\n dup 0 >\n { }\n { 0 swap - }\n if\n}", + "{\\n dup 0 >\\n { }\\n { 0 swap - }\\n if\\n}", [ Token(type="identifier", value="dup"), Token(type="i64", value=0), @@ -389,7 +359,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Comment at end of line inside token string self.make_token_string_test( "TokenString Comment End Of Line", - "{ 2 3 + // add them\n}", + "{ 2 3 + // add them\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), @@ -400,7 +370,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Multiple comments self.make_token_string_test( "TokenString Multiple Comments", - "{ 2 // first\n3 // second\n+ // add\n}", + "{ 2 // first\\n3 // second\\n+ // add\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), @@ -411,7 +381,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Comment on its own line self.make_token_string_test( "TokenString Comment Own Line", - "{\n // This is a comment\n 2 3 +\n}", + "{\\n // This is a comment\\n 2 3 +\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), @@ -422,7 +392,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Comment at start self.make_token_string_test( "TokenString Comment At Start", - "{ // comment\n2 3 + }", + "{ // comment\\n2 3 + }", [ Token(type="i64", value=2), Token(type="i64", value=3), @@ -433,7 +403,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Multiple comment lines self.make_token_string_test( "TokenString Multiple Comment Lines", - "{\n // First comment\n // Second comment\n 2 3 +\n}", + "{\\n // First comment\\n // Second comment\\n 2 3 +\\n}", [ Token(type="i64", value=2), Token(type="i64", value=3), @@ -444,7 +414,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Comments in nested token strings self.make_token_string_test( "TokenString Comments Nested", - "{ { 2 3 + // inner comment\n} // outer comment\n}", + "{ { 2 3 + // inner comment\\n} // outer comment\\n}", [ self.make_token_string_token([ Token(type="i64", value=2), @@ -545,14 +515,15 @@ class TokenStringTestGenerator(BaseTestGenerator): ) # If statement branches - self.make_token_string_test( - "TokenString If Branches", - '{ "positive" print }', - [ - Token(type="String", value="positive"), - Token(type="identifier", value="print") - ] - ) + if self.ENABLE_STRINGS: + self.make_token_string_test( + "TokenString If Branches", + '{ "positive" print }', + [ + Token(type="String", value="positive"), + Token(type="identifier", value="print") + ] + ) # Loop body self.make_token_string_test( @@ -601,7 +572,7 @@ class TokenStringTestGenerator(BaseTestGenerator): # Conditional with nested token strings self.make_token_string_test( "TokenString Conditional Complex", - "{\n dup 0 >\n { dup * }\n { drop 0 }\n if\n}", + "{\\n dup 0 >\\n { dup * }\\n { drop 0 }\\n if\\n}", [ Token(type="identifier", value="dup"), Token(type="i64", value=0), diff --git a/SLS_Tests/yaml_to_c_tests.py b/SLS_Tests/yaml_to_c_tests.py index d5d8f31..58d2cac 100644 --- a/SLS_Tests/yaml_to_c_tests.py +++ b/SLS_Tests/yaml_to_c_tests.py @@ -84,9 +84,162 @@ def _token_to_c_call(token: dict, idx_var="i") -> str: return f'test_boolean_value(&test, result, {idx_var}++, &(Boolean){{{"TRUE" if value else "FALSE"}}})' # type: ignore elif ttype == "error": return f'test_for_error(&test, result, i++, SLS_STR("{c_string_literal(value)}"))' # type: ignore + elif ttype == "token_string": + return _token_string_c_call(token, idx_var, value) # type: ignore else: raise ValueError(f' Unhandled token type: {ttype}') +def _token_string_c_call(token: dict, idx_var: str, value: list[dict]) -> str: + """Generate C code for testing a token string value.""" + # Build the array of TestTokenValue structures + token_values = [] + for inner_token in value: + inner_type = inner_token.get("type") + inner_value = inner_token.get("value") + + # Map token types to their C test handler and value structure + if inner_type == "i64": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I64, {inner_value}}}" + elif inner_type == "i32": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I32, {inner_value}}}" + elif inner_type == "i16": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I16, {inner_value}}}" + elif inner_type == "i8": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I8, {inner_value}}}" + elif inner_type == "u64": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U64, {inner_value}}}" + elif inner_type == "u32": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U32, {inner_value}}}" + elif inner_type == "u16": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U16, {inner_value}}}" + elif inner_type == "u8": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U8, {inner_value}}}" + elif inner_type == "f64": + handler = "test_double_value" + value_struct = f"&(double){{{inner_value}}}" + elif inner_type == "f32": + handler = "test_float_value" + value_struct = f"&(float){{{inner_value}}}" + elif inner_type == "char": + handler = "test_character_value" + value_struct = f"&(uint8_t){{{ord(inner_value)}}}" # type: ignore + elif inner_type == "string": + handler = "test_string_value" + value_struct = f'&SLS_STR("{c_string_literal(inner_value)}")' # type: ignore + elif inner_type == "identifier": + handler = "test_identifier_value" + value_struct = f'&(TestIdentifierValue){{FALSE, SLS_STR("{inner_value}")}}' + elif inner_type == "identifier_literal": + handler = "test_identifier_value" + value_struct = f'&(TestIdentifierValue){{TRUE, SLS_STR("{inner_value}")}}' + elif inner_type == "bool": + handler = "test_boolean_value" + bool_str = "TRUE" if inner_value else "FALSE" + value_struct = f"&(Boolean){{{bool_str}}}" + elif inner_type == "error": + handler = "test_for_error" + value_struct = f'SLS_STR("{c_string_literal(inner_value)}")' # type: ignore + elif inner_type == "token_string": + # Recursive case: nested token string + handler = "test_token_string_value" + nested_tokens = inner_value if isinstance(inner_value, list) else [] + value_struct = _build_token_string_value_struct(nested_tokens) + else: + raise ValueError(f'Unhandled inner token type in token_string: {inner_type}') + + token_values.append(f"{{.token_handler = {handler}, .value = {value_struct}}}") + + # Generate the TestTokenStringValue initialization + num_tokens = len(value) + if num_tokens == 0: + # Empty token string + return f'test_token_string_value(&test, result, {idx_var}, &(TestTokenStringValue){{0, NULL}})' + else: + # Token string with values + values_array = f"(TestTokenValue[]){{{', '.join(token_values)}}}" + return f'test_token_string_value(&test, result, {idx_var}, &(TestTokenStringValue){{{num_tokens}, {values_array}}})' + + +def _build_token_string_value_struct(nested_tokens: list[dict]) -> str: + """Helper to build a TestTokenStringValue structure for nested token strings.""" + if not nested_tokens: + return "&(TestTokenStringValue){0, NULL}" + + token_values = [] + for inner_token in nested_tokens: + inner_type = inner_token.get("type") + inner_value = inner_token.get("value") + + if inner_type == "i64": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I64, {inner_value}}}" + elif inner_type == "i32": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I32, {inner_value}}}" + elif inner_type == "i16": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I16, {inner_value}}}" + elif inner_type == "i8": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_I8, {inner_value}}}" + elif inner_type == "u64": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U64, {inner_value}}}" + elif inner_type == "u32": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U32, {inner_value}}}" + elif inner_type == "u16": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U16, {inner_value}}}" + elif inner_type == "u8": + handler = "test_integer_value" + value_struct = f"&(TestIntegerValue){{INTEGER_U8, {inner_value}}}" + elif inner_type == "f64": + handler = "test_double_value" + value_struct = f"&(double){{{inner_value}}}" + elif inner_type == "f32": + handler = "test_float_value" + value_struct = f"&(float){{{inner_value}}}" + elif inner_type == "char": + handler = "test_character_value" + value_struct = f"&(uint8_t){{{ord(inner_value)}}}" # type: ignore + elif inner_type == "string": + handler = "test_string_value" + value_struct = f'&SLS_STR("{c_string_literal(inner_value)}")' # type: ignore + elif inner_type == "identifier": + handler = "test_identifier_value" + value_struct = f'&(TestIdentifierValue){{FALSE, SLS_STR("{inner_value}")}}' + elif inner_type == "identifier_literal": + handler = "test_identifier_value" + value_struct = f'&(TestIdentifierValue){{TRUE, SLS_STR("{inner_value}")}}' + elif inner_type == "bool": + handler = "test_boolean_value" + bool_str = "TRUE" if inner_value else "FALSE" + value_struct = f"&(Boolean){{{bool_str}}}" + elif inner_type == "error": + handler = "test_for_error" + value_struct = f'SLS_STR("{c_string_literal(inner_value)}")' # type: ignore + elif inner_type == "token_string": + # Recursive case + handler = "test_token_string_value" + value_struct = _build_token_string_value_struct(inner_value if isinstance(inner_value, list) else []) + else: + raise ValueError(f'Unhandled nested token type in token_string: {inner_type}') + + token_values.append(f"{{.token_handler = {handler}, .value = {value_struct}}}") + + num_tokens = len(nested_tokens) + values_array = f"(TestTokenValue[]){{{', '.join(token_values)}}}" + return f"&(TestTokenStringValue){{{num_tokens}, {values_array}}}" + def token_to_c_call(token: dict, idx_var="i") -> str: """Generate a C 'test_*_value' call based on token type.""" return f"if ({_token_to_c_call(token, idx_var)}) return test.result;"