KylerOlsen/YREA-SLS
KylerOlsen
/
YREA-SLS
1
0
Fork 0
Code Releases 3 Wiki Activity

Added copy_token_string for properly making deep copies of token strings

This commit is contained in:
Kyler Olsen 2025-11-27 16:47:26 -07:00
parent 6f202602ec
commit 3512f00f30
3 changed files with 23 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
SLS_C/include/sls/lexer.h
View File

@ -154,6 +154,7 @@ typedef struct {
};
} LexerResult;
TokenString copy_token_string(TokenString token_string);
void init_lexer(LexerInfo *lexer_info, SlsStr filename, SlsStr source_code);
LexerTokenResult *get_token(LexerTokenResult *head, size_t i);
void clean_token_result(LexerTokenResult *head);

28
SLS_C/src/lexer.c
View File

@ -665,6 +665,26 @@ static void skip_comments_and_whitespace(LexerInfo *lexer_info) {
static LexerResult lexer_next(LexerInfo *lexer_info);
TokenString copy_token_string(TokenString token_string) {
TokenString new_string = (TokenString){
.length = token_string.length,
.tokens = (Token *)malloc(sizeof(Token) * token_string.length)
};
for (size_t i = 0; i < token_string.length; i++) {
if (token_string.tokens[i].type == TOKEN_STRING) {
new_string.tokens[i].type = TOKEN_STRING;
new_string.tokens[i].string_literal = sls_str_cpy(token_string.tokens[i].string_literal);
} else if (token_string.tokens[i].type == TOKEN_TOKEN_STRING) {
new_string.tokens[i].type = TOKEN_TOKEN_STRING;
new_string.tokens[i].token_string = copy_token_string(token_string.tokens[i].token_string);
} else {
new_string.tokens[i] = token_string.tokens[i];
}
}
return new_string;
}
static LexerResult convert_to_token_string(LexerInfo *lexer_info, LexerTokenResult *head, size_t start, size_t start_line) {
TokenString token_string = {
.length = 0,
@ -685,14 +705,8 @@ static LexerResult convert_to_token_string(LexerInfo *lexer_info, LexerTokenResu
token_string.tokens[i].type = TOKEN_STRING;
token_string.tokens[i].string_literal = sls_str_cpy(current->result.string_literal);
} else if (current->result.type == TOKEN_TOKEN_STRING) {
// TODO: This is only a half deep copy
// Any token strings inside the token string are not copied correctly
// A token string deep copy function is likely in order
token_string.tokens[i].type = TOKEN_TOKEN_STRING;
token_string.tokens[i].token_string = (TokenString){
.length = current->result.token_string.length,
.tokens = (Token *)malloc(sizeof(Token) * current->result.token_string.length)
};
token_string.tokens[i].token_string = copy_token_string(current->result.token_string);
memcpy(token_string.tokens[i].token_string.tokens, current->result.token_string.tokens, current->result.token_string.length);
} else token_string.tokens[i] = current->result;
current = current->next;

5
SLS_C/tests/lexer_test_helpers.c
View File

@ -512,10 +512,7 @@ static LexerResult token_string_to_lexer_result(TokenString token_string, FileIn
.type = SLS_RESULT,
.result = (Token){
.type = TOKEN_TOKEN_STRING,
.token_string = (TokenString){
.length = token_string.tokens[i-1].token_string.length,
.tokens = (Token *)malloc(sizeof(Token) * token_string.tokens[i-1].token_string.length),
}
.token_string = copy_token_string(token_string.tokens[i-1].token_string)
},
.file_info = file_info,
.next = head