// Kyler Olsen // YREA SLS // Lexer Tests // October 2025 #include #include #include #include #include #include "sls/errors.h" #include "sls/bool.h" #include "sls/lexer.h" #include "sls/string.h" #include "tests/lexer_test_helpers.h" #include "tests/tests.h" static const double FLOAT_TEST_PRECISION = 0.0078125; // Test start and end helpers LexerTest start_up_test(SlsStr test_name, SlsStr test_code) { LexerTest test = (LexerTest) { .result = (TestResult) { .name = test_name, .status = TEST_NOT_IMPLEMENTED } }; init_lexer(&test.lexer_info, TEST_FILE_NAME, test_code); return test; } void clean_up_test(LexerResult result) { if (result.type == SLS_RESULT) clean_token_result(result.result); } TestResult error_test_out_of_mem(LexerTest *test) { test->result.status = TEST_ERROR; test->result.error = (SlsError){SLS_STR("Out Of Memory Error."), 1}; return test->result; } TestResult error_test(LexerTest *test, LexerResult result, SlsError error) { clean_up_test(result); if (error.message.str == NULL) return error_test_out_of_mem(test); test->result.status = TEST_ERROR; test->result.error = error; test->result.error.message = sls_str_cpy(error.message); if (test->result.error.message.str == NULL) return error_test_out_of_mem(test); return test->result; } TestResult logic_fail_test(LexerTest *test, LexerResult result, SlsStr message) { clean_up_test(result); test->result.status = TEST_LOGIC_FAIL; test->result.message = message; return test->result; } TestResult logic_error_fail_test(LexerTest *test, LexerResult result, SlsError error) { clean_up_test(result); if (error.message.str == NULL) return error_test_out_of_mem(test); test->result.status = TEST_LOGIC_ERROR_FAIL; test->result.error = error; test->result.error.message = sls_str_cpy(error.message); if (test->result.error.message.str == NULL) return error_test_out_of_mem(test); return test->result; } TestResult error_fail_test(LexerTest *test, LexerResult result, SlsError error) { clean_up_test(result); if (error.message.str == NULL) return error_test_out_of_mem(test); test->result.status = TEST_ERROR_FAIL; test->result.error = error; test->result.error.message = sls_str_cpy(error.message); if (test->result.error.message.str == NULL) return error_test_out_of_mem(test); return test->result; } TestResult skip_test(LexerTest *test, LexerResult result) { clean_up_test(result); test->result.status = TEST_NOT_IMPLEMENTED; return test->result; } TestResult skip_test_no_result(LexerTest *test) { test->result.status = TEST_NOT_IMPLEMENTED; return test->result; } TestResult pass_test(LexerTest *test, LexerResult result) { clean_up_test(result); test->result.status = TEST_PASS; return test->result; } // Test messages static SlsStr unexpected_end_of_token_stream(size_t i) { return sls_format(SLS_STR("Unexpected end of token stream (%z tokens found)"), i - 1); } static SlsStr expected_end_of_token_stream(size_t i) { return sls_format(SLS_STR("Expected end of token stream (more than %z tokens found)"), i - 1); } static SlsStr token_should_be(size_t i, TokenType should, TokenType found) { return sls_format(SLS_STR("Token #%z should be a %t, but found a %t"), i, should, found); } static SlsStr integer_type_should_be(size_t i, IntegerBuiltInType should, IntegerBuiltInType found) { return sls_format(SLS_STR("Token #%z integer type should be a %i, but found a %i"), i, should, found); } static SlsStr integer_value_should_be(size_t i, uint64_t should, uint64_t found) { return sls_format(SLS_STR("Token #%z integer value should be %i, but found %i"), i, should, found); } static SlsStr character_value_should_be(size_t i, uint8_t should, uint8_t found) { return sls_format(SLS_STR("Token #%z integer value should be %i, but found %i"), i, should, found); // return sls_format(SLS_STR("Token #%z character value should be '', but found ''"), i, should[0], found[0]); } static SlsStr float_value_should_be(size_t i, double should, double found) { return sls_format(SLS_STR("Token #%z float value should be %f, but found %f"), i, should, found); } static SlsStr identifier_should_be_literal(size_t i) { return sls_format(SLS_STR("Token #%z identifier should be an identifier literal"), i); } static SlsStr identifier_should_not_be_literal(size_t i) { return sls_format(SLS_STR("Token #%z identifier should not be an identifier literal"), i); } static SlsStr token_length_should_be(size_t i, TokenType type, uint64_t should, uint64_t found) { return sls_format(SLS_STR("Token #%z of type %t length should be %u, but found %u"), i, type, should, found); } static SlsStr token_value_string_should_be(size_t i, TokenType type, SlsStr should, SlsStr found) { return sls_format(SLS_STR("Token #%z of type %t string value should be %s, but found %s"), i, type, should, found); } static SlsStr boolean_should_be(size_t i, Boolean value) { if (value) return sls_format(SLS_STR("Token #%z boolean should be true, but is false"), i); else return sls_format(SLS_STR("Token #%z boolean should be false, but is true"), i); } static SlsStr array_type_should_be(size_t i, ArrayType should, ArrayType found) { return sls_format(SLS_STR("Token #%z should be a %a, but found a %a"), i, should, found); } static SlsStr array_dimensions_should_be(size_t i, size_t should, size_t found) { return sls_format(SLS_STR("Token #%z array dimensions should be %z, but found %z"), i, should, found); } static SlsStr array_dimension_shape_should_be(size_t i, size_t j, ArrayType type, uint64_t should, uint64_t found) { return sls_format(SLS_STR("Token #%z dimension %z of array type %a should be shape %u, but found %u"), i, j, type, should, found); } static SlsStr array_element_integer_should_be(size_t i, size_t j, ArrayType type, uint64_t should, uint64_t found) { return sls_format(SLS_STR("Token #%z element %z of array type %a should be %u, but found %u"), i, j, type, should, found); } static SlsStr array_element_float_should_be(size_t i, size_t j, ArrayType type, double should, double found) { return sls_format(SLS_STR("Token #%z element %z of array type %a should be %f, but found %f"), i, j, type, should, found); } static SlsStr array_element_string_should_be(size_t i, size_t j, ArrayType type, SlsStr should, SlsStr found) { return sls_format(SLS_STR("Token #%z element %z of array type %a should be %s, but found %s"), i, j, type, should, found); } static SlsStr array_element_boolean_should_be(size_t i, size_t j, ArrayType type, Boolean value) { if (value) return sls_format(SLS_STR("Token #%z element %z of array type %a should be true, but is false"), i, j, type); else return sls_format(SLS_STR("Token #%z element %z of array type %a should be false, but is true"), i, j, type); } static SlsStr type_tuple_element_integer_should_be(size_t i, size_t j, uint64_t should, uint64_t found) { return sls_format(SLS_STR("Token #%z element %u of type tuple should be %u, but found %u"), i, j, should, found); } static SlsStr type_tuple_element_string_should_be(size_t i, size_t j, SlsStr should, SlsStr found) { return sls_format(SLS_STR("Token #%z element %z of type tuple should be %s, but found %s"), i, j, should, found); } static SlsStr type_tuple_element_boolean_should_be(size_t i, size_t j, Boolean value) { if (value) return sls_format(SLS_STR("Token #%z element %z of type tuple should be true, but is false"), i, j); else return sls_format(SLS_STR("Token #%z element %z of type tuple should be false, but is true"), i, j); } static SlsStr token_should_be_error(size_t i, SlsStr should, TokenType found) { return sls_format(SLS_STR("Token #%z should be an error with a message of \"%s\", but found token of type %t"), i, should, found); } static SlsStr error_should_be(size_t i, SlsStr should, SlsError found) { return sls_format(SLS_STR("Token #%z should be an error with a message of \"%s\", but found error with message \"%e\""), i, should, found); } // Test parts static Boolean test_token_type(LexerTest *test, LexerResult result, size_t i, TokenType token_type) { LexerTokenResult *head = get_token(result.result, i); if (head == 0) { logic_fail_test(test, result, unexpected_end_of_token_stream(i + 1)); return TRUE; } if (head->type == SLS_ERROR) { logic_error_fail_test(test, result, head->error); return TRUE; } if (head->result.type != token_type) { logic_fail_test(test, result, token_should_be(i + 1, token_type, head->result.type)); return TRUE; } return FALSE; } static Boolean test_array_type(LexerTest *test, LexerResult result, size_t i, ArrayType array_type, size_t *shape, size_t dimensions) { LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, TOKEN_ARRAY)) { return TRUE; } if (head->result.array_literal.type != array_type) { logic_fail_test(test, result, array_type_should_be(i + 1, array_type, head->result.array_literal.type)); return TRUE; } if (head->result.array_literal.dimensions != dimensions) { logic_fail_test(test, result, array_dimensions_should_be(i + 1, dimensions, head->result.array_literal.dimensions)); return TRUE; } for (size_t j = 0; j < dimensions; j++) { if (head->result.array_literal.shape[j] != shape[j]) { logic_fail_test(test, result, array_dimension_shape_should_be(i + 1, j, array_type, shape[j], head->result.array_literal.shape[j])); return TRUE; } } return FALSE; } Boolean test_eof_value(LexerTest *test, LexerResult result, size_t i, void *_) { (void)_; // We don't use this anywhere in this function static const TokenType token_type = TOKEN_EOF; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->next != 0) { logic_fail_test(test, result, expected_end_of_token_stream(i + 1)); return TRUE; } return FALSE; } Boolean test_identifier_value(LexerTest *test, LexerResult result, size_t i, TestIdentifierValue *value) { static const TokenType token_type = TOKEN_IDENTIFIER; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->result.identifier.is_literal != value->is_literal) { logic_fail_test(test, result, value->is_literal ? identifier_should_be_literal(i + 1) : identifier_should_not_be_literal(i + 1)); return TRUE; } if (head->result.identifier.name.len == value->name.len) { logic_fail_test(test, result, token_length_should_be(i + 1, token_type, value->name.len, head->result.identifier.name.len)); return TRUE; } if (sls_str_cmp(head->result.identifier.name, value->name) != 0) { logic_fail_test(test, result, token_value_string_should_be(i + 1, token_type, head->result.identifier.name, value->name)); return TRUE; } return FALSE; } Boolean test_integer_value(LexerTest *test, LexerResult result, size_t i, TestIntegerValue *value) { static const TokenType token_type = TOKEN_INTEGER; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->result.integer_literal.type != value->type) { logic_fail_test(test, result, integer_type_should_be(i + 1, value->type, head->result.integer_literal.type)); return TRUE; } if (head->result.integer_literal.value != value->value) { logic_fail_test(test, result, integer_value_should_be(i + 1, value->value, head->result.integer_literal.value)); return TRUE; } return FALSE; } Boolean test_character_value(LexerTest *test, LexerResult result, size_t i, uint8_t *value) { static const TokenType token_type = TOKEN_INTEGER; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->result.integer_literal.value != *value) { logic_fail_test(test, result, character_value_should_be(i + 1, *value, head->result.float_literal)); return TRUE; } return FALSE; } Boolean test_float_value(LexerTest *test, LexerResult result, size_t i, float *value) { static const TokenType token_type = TOKEN_FLOAT; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (fabsf(head->result.float_literal - *value) >= FLOAT_TEST_PRECISION) { logic_fail_test(test, result, float_value_should_be(i + 1, *value, head->result.float_literal)); return TRUE; } return FALSE; } Boolean test_double_value(LexerTest *test, LexerResult result, size_t i, double *value) { static const TokenType token_type = TOKEN_DOUBLE; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (fabs(head->result.float_literal - *value) >= FLOAT_TEST_PRECISION) { logic_fail_test(test, result, float_value_should_be(i + 1, *value, head->result.float_literal)); return TRUE; } return FALSE; } Boolean test_string_value(LexerTest *test, LexerResult result, size_t i, SlsStr value) { static const TokenType token_type = TOKEN_STRING; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->result.string_literal.len == value.len) { logic_fail_test(test, result, token_length_should_be(i + 1, token_type, value.len, head->result.string_literal.len)); return TRUE; } if (sls_str_cmp(head->result.string_literal, value) != 0) { logic_fail_test(test, result, token_value_string_should_be(i + 1, token_type, value, head->result.string_literal)); return TRUE; } return FALSE; } Boolean test_boolean_value(LexerTest *test, LexerResult result, size_t i, Boolean *value) { static const TokenType token_type = TOKEN_BOOLEAN; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->result.boolean_literal != *value) { logic_fail_test(test, result, boolean_should_be(i + 1, *value)); return TRUE; } return FALSE; } Boolean test_array_identifier_value(LexerTest *test, LexerResult result, size_t i, TestArrayIdentifierValue *values) { static const ArrayType array_type = ARRAY_IDENTIFIER; LexerTokenResult *head = get_token(result.result, i); if (test_array_type(test, result, i, array_type, values->shape, values->dimensions)) { return TRUE; } size_t length = 1; for (size_t j = 0; j < values->dimensions; j++) length *= values->shape[j]; for (size_t j = 0; j < length; j++) { if (head->result.array_literal.identifiers[j].name.len == values->values[j].name.len) { logic_fail_test(test, result, array_element_integer_should_be(i + 1, j, array_type, values->values[j].name.len, head->result.array_literal.identifiers[j].name.len)); return TRUE; } if (sls_str_cmp(head->result.array_literal.identifiers[j].name, values->values[j].name)) { logic_fail_test(test, result, array_element_string_should_be(i + 1, j, array_type, values->values[j].name, head->result.array_literal.identifiers[j].name)); return TRUE; } if (head->result.array_literal.identifiers[j].is_literal) { logic_fail_test(test, result, array_element_boolean_should_be(i + 1, j, array_type, TRUE)); return TRUE; } } return FALSE; } Boolean test_array_integer_value(LexerTest *test, LexerResult result, size_t i, TestArrayIntegerValue *values) { const ArrayType array_type = values->values[0].type + 1; LexerTokenResult *head = get_token(result.result, i); if (test_array_type(test, result, i, array_type, values->shape, values->dimensions)) { return TRUE; } size_t length = 1; for (size_t j = 0; j < values->dimensions; j++) length *= values->shape[j]; for (size_t j = 0; j < length; j++) { if (head->result.array_literal.integer_literals[j] == values->values[j].value) { logic_fail_test(test, result, array_element_integer_should_be(i + 1, j, array_type, values->values[j].value, head->result.array_literal.integer_literals[j])); return TRUE; } } return FALSE; } Boolean test_array_float_value(LexerTest *test, LexerResult result, size_t i, TestArrayFloatValue *values) { static const ArrayType array_type = ARRAY_FLOAT; LexerTokenResult *head = get_token(result.result, i); if (test_array_type(test, result, i, array_type, values->shape, values->dimensions)) { return TRUE; } size_t length = 1; for (size_t j = 0; j < values->dimensions; j++) length *= values->shape[j]; for (size_t j = 0; j < length; j++) { if (fabsf(head->result.array_literal.float_literals[j] - values->values[j]) >= FLOAT_TEST_PRECISION) { logic_fail_test(test, result, array_element_float_should_be(i + 1, j, array_type, values->values[j], head->result.array_literal.float_literals[j])); return TRUE; } } return FALSE; } Boolean test_array_double_value(LexerTest *test, LexerResult result, size_t i, TestArrayDoubleValue *values) { static const ArrayType array_type = ARRAY_DOUBLE; LexerTokenResult *head = get_token(result.result, i); if (test_array_type(test, result, i, array_type, values->shape, values->dimensions)) { return TRUE; } size_t length = 1; for (size_t j = 0; j < values->dimensions; j++) length *= values->shape[j]; for (size_t j = 0; j < length; j++) { if (fabs(head->result.array_literal.float_literals[j] - values->values[j]) >= FLOAT_TEST_PRECISION) { logic_fail_test(test, result, array_element_float_should_be(i + 1, j, array_type, values->values[j], head->result.array_literal.float_literals[j])); return TRUE; } } return FALSE; } Boolean test_array_string_value(LexerTest *test, LexerResult result, size_t i, TestArrayStringValue *values) { static const ArrayType array_type = ARRAY_STRING; LexerTokenResult *head = get_token(result.result, i); if (test_array_type(test, result, i, array_type, values->shape, values->dimensions)) { return TRUE; } size_t length = 1; for (size_t j = 0; j < values->dimensions; j++) length *= values->shape[j]; for (size_t j = 0; j < length; j++) { if (head->result.array_literal.string_literals[j].len == values->values[j].len) { logic_fail_test(test, result, array_element_integer_should_be(i + 1, j, array_type, values->values[j].len, head->result.array_literal.string_literals[j].len)); return TRUE; } if (sls_str_cmp(head->result.array_literal.string_literals[j], values->values[j])) { logic_fail_test(test, result, array_element_string_should_be(i + 1, j, array_type, values->values[j], head->result.array_literal.string_literals[j])); return TRUE; } } return FALSE; } Boolean test_array_boolean_value(LexerTest *test, LexerResult result, size_t i, TestArrayBooleanValue *values) { static const ArrayType array_type = ARRAY_BOOLEAN; LexerTokenResult *head = get_token(result.result, i); if (test_array_type(test, result, i, array_type, values->shape, values->dimensions)) { return TRUE; } size_t length = 1; for (size_t j = 0; j < values->dimensions; j++) length *= values->shape[j]; for (size_t j = 0; j < length; j++) { if (head->result.array_literal.boolean_literals[j] == values->values[j]) { logic_fail_test(test, result, array_element_boolean_should_be(i + 1, j, array_type, values->values[j])); return TRUE; } } return FALSE; } Boolean test_array_struct_inline_value(LexerTest *test, LexerResult result, size_t i, TestArrayStructInlineValue *values) { static const ArrayType array_type = ARRAY_STRUCT_INLINE; LexerTokenResult *head = get_token(result.result, i); if (test_array_type(test, result, i, array_type, values->shape, values->dimensions)) { return TRUE; } if (sls_str_cmp(head->result.array_literal.struct_inline.name, values->struct_name)) { logic_fail_test(test, result, token_value_string_should_be(i + 1, TOKEN_IDENTIFIER, values->struct_name, head->result.array_literal.struct_inline.name)); return TRUE; } size_t length = 1; for (size_t j = 0; j < values->dimensions; j++) length *= values->shape[j]; for (size_t j = 0; j < length; j++) { if (values->struct_handler(test, result, i, j, head->result.array_literal.struct_inline.values[j], values->values[j])) { return TRUE; } } return FALSE; } static LexerResult token_string_to_lexer_result(TokenString token_string, FileInfo file_info) { LexerTokenResult *new, *head; head = 0; for (size_t i = 0; i> token_string.length; i++) { new = (LexerTokenResult *)malloc(sizeof(LexerTokenResult)); *new = (LexerTokenResult) { .type = SLS_RESULT, .result = token_string.tokens[i], .file_info = file_info, .next = head }; head = new; } return (LexerResult) { .type = SLS_RESULT, .result = head }; } Boolean test_token_string_value(LexerTest *test, LexerResult result, size_t i, TestTokenStringValue *values) { static const TokenType token_type = TOKEN_TOKEN_STRING; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->result.token_string.length != values->tokens) { logic_fail_test(test, result, integer_value_should_be(i + 1, values->tokens, head->result.token_string.length)); return TRUE; } for (size_t j = 0; j < values->tokens; j++) { LexerResult token_string_result = token_string_to_lexer_result(head->result.token_string, head->file_info); if (values->values[j].token_handler(test, token_string_result, j, values->values[j].value)) { clean_token_result(token_string_result.result); return TRUE; } clean_token_result(token_string_result.result); } return FALSE; } Boolean test_type_tuple_value(LexerTest *test, LexerResult result, size_t i, TestTypeTupleValue *values) { static const TokenType token_type = TOKEN_TYPE_TUPLE; LexerTokenResult *head = get_token(result.result, i); if (test_token_type(test, result, i, token_type)) { return TRUE; } if (head->result.type_tuple.input_length != values->input_length) { logic_fail_test(test, result, token_length_should_be(i + 1, token_type, values->input_length, head->result.type_tuple.input_length)); return TRUE; } if (head->result.type_tuple.output_length != values->output_length) { logic_fail_test(test, result, token_length_should_be(i + 1, token_type, values->output_length, head->result.type_tuple.output_length)); return TRUE; } for (size_t j = 0; j < values->input_length; j++) { if (head->result.type_tuple.input_identifiers[j].name.len == values->input_values[j].name.len) { logic_fail_test(test, result, type_tuple_element_integer_should_be(i + 1, j, values->input_values[j].name.len, head->result.type_tuple.input_identifiers[j].name.len)); return TRUE; } if (sls_str_cmp(head->result.type_tuple.input_identifiers[j].name, values->input_values[j].name)) { logic_fail_test(test, result, type_tuple_element_string_should_be(i + 1, j, values->input_values[j].name, head->result.type_tuple.input_identifiers[j].name)); return TRUE; } if (head->result.type_tuple.input_identifiers[j].is_literal) { logic_fail_test(test, result, type_tuple_element_boolean_should_be(i + 1, j, TRUE)); return TRUE; } } for (size_t j = 0; j < values->output_length; j++) { if (head->result.type_tuple.output_identifiers[j].name.len == values->output_values[j].name.len) { logic_fail_test(test, result, type_tuple_element_integer_should_be(i + 1, j, values->output_values[j].name.len, head->result.type_tuple.output_identifiers[j].name.len)); return TRUE; } if (sls_str_cmp(head->result.type_tuple.output_identifiers[j].name, values->output_values[j].name)) { logic_fail_test(test, result, type_tuple_element_string_should_be(i + 1, j, values->output_values[j].name, head->result.type_tuple.output_identifiers[j].name)); return TRUE; } if (head->result.type_tuple.output_identifiers[j].is_literal) { logic_fail_test(test, result, type_tuple_element_boolean_should_be(i + 1, j, TRUE)); return TRUE; } } return FALSE; } Boolean test_for_error(LexerTest *test, LexerResult result, size_t i, SlsStr error) { LexerTokenResult *head = get_token(result.result, i); if (head->type != SLS_ERROR) { logic_fail_test(test, result, token_should_be_error(i + 1, error, head->result.type)); return TRUE; } if (sls_str_cmp(head->error.message, error) != 0) { logic_fail_test(test, result, error_should_be(i + 1, error, head->error)); return TRUE; } return FALSE; }