diff --git a/SLS_C/include/sls/lexer.h b/SLS_C/include/sls/lexer.h index b53f068..7509995 100644 --- a/SLS_C/include/sls/lexer.h +++ b/SLS_C/include/sls/lexer.h @@ -154,6 +154,7 @@ typedef struct { }; } LexerResult; +TokenString copy_token_string(TokenString token_string); void init_lexer(LexerInfo *lexer_info, SlsStr filename, SlsStr source_code); LexerTokenResult *get_token(LexerTokenResult *head, size_t i); void clean_token_result(LexerTokenResult *head); diff --git a/SLS_C/src/lexer.c b/SLS_C/src/lexer.c index d012197..3a5d90a 100644 --- a/SLS_C/src/lexer.c +++ b/SLS_C/src/lexer.c @@ -665,6 +665,26 @@ static void skip_comments_and_whitespace(LexerInfo *lexer_info) { static LexerResult lexer_next(LexerInfo *lexer_info); +TokenString copy_token_string(TokenString token_string) { + TokenString new_string = (TokenString){ + .length = token_string.length, + .tokens = (Token *)malloc(sizeof(Token) * token_string.length) + }; + + for (size_t i = 0; i < token_string.length; i++) { + if (token_string.tokens[i].type == TOKEN_STRING) { + new_string.tokens[i].type = TOKEN_STRING; + new_string.tokens[i].string_literal = sls_str_cpy(token_string.tokens[i].string_literal); + } else if (token_string.tokens[i].type == TOKEN_TOKEN_STRING) { + new_string.tokens[i].type = TOKEN_TOKEN_STRING; + new_string.tokens[i].token_string = copy_token_string(token_string.tokens[i].token_string); + } else { + new_string.tokens[i] = token_string.tokens[i]; + } + } + return new_string; +} + static LexerResult convert_to_token_string(LexerInfo *lexer_info, LexerTokenResult *head, size_t start, size_t start_line) { TokenString token_string = { .length = 0, @@ -685,14 +705,8 @@ static LexerResult convert_to_token_string(LexerInfo *lexer_info, LexerTokenResu token_string.tokens[i].type = TOKEN_STRING; token_string.tokens[i].string_literal = sls_str_cpy(current->result.string_literal); } else if (current->result.type == TOKEN_TOKEN_STRING) { - // TODO: This is only a half deep copy - // Any token strings inside the token string are not copied correctly - // A token string deep copy function is likely in order token_string.tokens[i].type = TOKEN_TOKEN_STRING; - token_string.tokens[i].token_string = (TokenString){ - .length = current->result.token_string.length, - .tokens = (Token *)malloc(sizeof(Token) * current->result.token_string.length) - }; + token_string.tokens[i].token_string = copy_token_string(current->result.token_string); memcpy(token_string.tokens[i].token_string.tokens, current->result.token_string.tokens, current->result.token_string.length); } else token_string.tokens[i] = current->result; current = current->next; diff --git a/SLS_C/tests/lexer_test_helpers.c b/SLS_C/tests/lexer_test_helpers.c index 04f1950..91d1665 100644 --- a/SLS_C/tests/lexer_test_helpers.c +++ b/SLS_C/tests/lexer_test_helpers.c @@ -512,10 +512,7 @@ static LexerResult token_string_to_lexer_result(TokenString token_string, FileIn .type = SLS_RESULT, .result = (Token){ .type = TOKEN_TOKEN_STRING, - .token_string = (TokenString){ - .length = token_string.tokens[i-1].token_string.length, - .tokens = (Token *)malloc(sizeof(Token) * token_string.tokens[i-1].token_string.length), - } + .token_string = copy_token_string(token_string.tokens[i-1].token_string) }, .file_info = file_info, .next = head