archeus/src/std/lexer.c

#include "arc/std/lexer.h"

#include "arc/std/bool.h"
#include "arc/std/errno.h"
#include "arc/std/string.h"
#include "arc/std/vector.h"
#include "arc/std/io.h"
#include <stdlib.h>

struct ARC_Lexer {
    ARC_Vector *tokenRules;
    ARC_Vector *tokens;
};

//private function for checking if two lexer token rules are the same in a vector (based on id)
ARC_Bool ARC_LexerTokenRule_VectorCompareDataFn(void *dataA, void *dataB){
    ARC_LexerTokenRule *tokenRuleA = (ARC_LexerTokenRule *)dataA;
    ARC_LexerTokenRule *tokenRuleB = (ARC_LexerTokenRule *)dataB;

    if(tokenRuleA->id == tokenRuleB->id){
        return ARC_True;
    }

    return ARC_False;
}

//private function for destroying a lexer token rule from a vector
void ARC_LexerTokenRule_VectorDestroyDataFn(void *data){
    ARC_LexerTokenRule *tokenRule = (ARC_LexerTokenRule *)data;
    tokenRule->destroyAutomataDataFn(tokenRule->automataData);
    free(tokenRule);
}


//private function for destroying a lexer token from a vector
void ARC_LexerToken_VectorDestroyDataFn(void *data){
    ARC_LexerToken *token = (ARC_LexerToken *)data;
    free(token);
}

void ARC_Lexer_Create(ARC_Lexer **lexer){
    //create the lexer
    *lexer = (ARC_Lexer *)malloc(sizeof(ARC_Lexer));

    //setup token rules vector with compare and delete functions
    ARC_Vector_CompareDataFn tokenRulesVectorCompareDataFn = ARC_LexerTokenRule_VectorCompareDataFn;
    ARC_Vector_DestroyDataFn tokenRulesVectorDestroyDataFn = ARC_LexerTokenRule_VectorDestroyDataFn;
    ARC_Vector_Create(&(*lexer)->tokenRules, &tokenRulesVectorCompareDataFn, &tokenRulesVectorDestroyDataFn);

    //setup tokens vector with delete funtion, we don't want a deleteDataFn because their index will be used as the id
    ARC_Vector_DestroyDataFn tokenVectorDestroyDataFn = ARC_LexerToken_VectorDestroyDataFn;
    ARC_Vector_Create(&(*lexer)->tokens, NULL, &tokenVectorDestroyDataFn);
}

void ARC_Lexer_Destroy(ARC_Lexer *lexer){
    //free the tokens (there is a vectorDeleteDataFn, so tokens should be freed)
    ARC_Vector_Destroy(lexer->tokens);

    //free the token rules (there is a vectorDeleteDataFn, so token rules should be freed)
    ARC_Vector_Destroy(lexer->tokenRules);

    //free the lexer
    free(lexer);
}

void ARC_Lexer_RegisterTokenRule(ARC_Lexer *lexer, ARC_LexerTokenRule tokenRule){
    //copy the token to a new pointer
    ARC_LexerTokenRule *storedTokenRule = (ARC_LexerTokenRule *)malloc(sizeof(ARC_LexerTokenRule));
    *storedTokenRule = tokenRule;

    //add to the vector and check for error (I'd be surprised if the error ever happened because that would most likely mean overflow)
    ARC_Vector_Add(lexer->tokenRules, storedTokenRule);
    if(arc_errno){
        ARC_DEBUG_LOG_ERROR("ARC_Lexer_RegisterTokenRule(lexer, tokenRule), errored when running ARC_Vector_Add(lexer->tokenRules, storedTokenRule);. check logs for more info");
        free(storedTokenRule);
    }
}

void ARC_Lexer_LexString(ARC_Lexer *lexer, ARC_String **data){
    //check if there are any token rules to use
    if(ARC_Vector_GetSize(lexer->tokenRules) == 0){
        arc_errno = ARC_ERRNO_DATA;
        ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexString(lexer, data), no tokens registered to lexer to use");
        return;
    }

    //this will run untill everything token is stripped or there is an error
    while(*data != NULL){
        uint32_t tokenLength = 0;
        uint32_t lastTokenLength = 0;
        ARC_LexerToken *token = NULL;

        for(uint32_t index = 0; index < ARC_Vector_GetSize(lexer->tokenRules); index++){
            //check if the token rule is found
            ARC_LexerTokenRule *tokenRule = ARC_Vector_Get(lexer->tokenRules, index);

            //set the last token length if the last token had a length
            if(tokenLength > 0){
                lastTokenLength = tokenLength;
            }

            //tokenData should only exist if tokenLength is ARC_True as stated in the header
            ARC_String *tokenData;
            tokenLength = tokenRule->automataFn(&tokenData, *data, tokenRule->automataData);

            //check if a token was found if it wasn't continue. I'm doing this to try to cut down on the ammount of indentation
            if(tokenLength == 0){
                continue;
            }

            //check to see if we found a better match
            if(tokenLength > lastTokenLength){
                //free the current token if it exists
                if(token != NULL){
                    ARC_LexerTokenRule_VectorDestroyDataFn((void *)token);
                }

                //create the token to add
                token = (ARC_LexerToken *)malloc(sizeof(ARC_LexerToken));
                token->rule = tokenRule->id;
                token->data = tokenData;
            }
        }

        //if no token was found, throw an error
        if(token == NULL){
            arc_errno = ARC_ERRNO_DATA;
            ARC_DEBUG_LOG_ERROR_WITH_VARIABLES("ARC_Lexer_LexString(lexer, data), no tokens found with current string: \"%s\"", (*data)->data);

            //clean up errored string
            ARC_String_Destroy(*data);
            *data = NULL;
            return;
        }

        //token exists (something must have gone very wrong if it doesn't), so add it and check for overflow (which I'd be surprised if that happens)
        ARC_Vector_Add(lexer->tokens, (void *)token);
        if(arc_errno){
            ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexString(lexer, data), errored when running ARC_Vector_Add(lexer->tokens, token);. check logs for more info");
            free(token);

            //clean up errored string
            ARC_String_Destroy(*data);
            *data = NULL;
            return;
        }

        //if the last token was found, destroy the string and return
        if(lastTokenLength == (*data)->length){
            ARC_String_Destroy(*data);
            *data = NULL;
            return;
        }

        //strip the string
        ARC_String_ReplaceWithSubstring(data, lastTokenLength, (*data)->length - lastTokenLength);
    }
}

void ARC_Lexer_LexFile(ARC_Lexer *lexer, ARC_String *path){
    //read file and clean up if it errors
    ARC_String *data;
    ARC_IO_FileToStr(path, &data);
    if(arc_errno){
        ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexFile(lexer, path), errored when running ARC_IO_FileToStr(path, &data);. check logs for more info");
        if(data != NULL){
            ARC_String_Destroy(data);
        }

        return;
    }

    //lex the string and log if there is an error, ARC_Lexer_LexString will clean up the string
    ARC_Lexer_LexString(lexer, &data);
    if(arc_errno){
        ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexFile(lexer, path), errored when running ARC_Lexer_LexString(lexer, data);. check logs for more info");
    }
}

ARC_LexerToken ARC_Lexer_GetToken(ARC_Lexer *lexer, uint32_t index){
    //get the token and log if there is an error
    ARC_LexerToken *token = ARC_Vector_Get(lexer->tokens, index);
    if(arc_errno){
        ARC_DEBUG_LOG_ERROR("ARC_Lexer_GetToken(lexer, index), errored when running ARC_Vector_Get(lexer->tokens, index);. check logs for more info");

        //return a token with max rule value, and NULL for the string to signify an error
        return (ARC_LexerToken){
            ~(uint32_t)0,
            NULL
        };
    }

    //the token was found, so return a copy to that
    return *token;
}

uint32_t ARC_Lexer_GetTokensSize(ARC_Lexer *lexer){
    return ARC_Vector_GetSize(lexer->tokens);
}

uint32_t ARC_Lexer_AutomataMatchCharFn(ARC_String **tokenData, ARC_String *string, void *automataData){
    //if there is a match the token will be the same as automataData, so we don't need to store it again
    *tokenData = NULL;

    //check to see if there is a match with automataData as a char
    if(string->data[0] == *(char *)automataData){
        //return the token was found of length 1
        return 1;
    }

    //no match was found
    return 0;
}

uint32_t ARC_Lexer_AutomataMatchStringFn(ARC_String **tokenData, ARC_String *string, void *automataData){
    //if there is a match the token will be the same as automataData, so we don't need to store it again
    *tokenData = NULL;

    //check to see if there is a match with automataData as a string
    ARC_String *automataDataString = (ARC_String *)automataData;
    if(ARC_String_SubstringEquals(string, 0, automataDataString)){
        //return the token was found of the string length
        return automataDataString->length;
    }

    //no match was found
    return 0;
}

uint32_t ARC_Lexer_AutomataMatchCharInStringFn(ARC_String **tokenData, ARC_String *string, void *automataData){
    //if there is a match the token will be the same as automataData, so we don't need to store it again
    *tokenData = NULL;

    //check to see if there is a char match in automataData as a string
    ARC_String *automataDataString = (ARC_String *)automataData;
    for(uint64_t index = 0; index < automataDataString->length; index++){
        if(string->data[0] == automataDataString->data[index]){
            //return the token was found in the string of length 1
            return 1;
        }
    }

    //no match was found
    return 0;
}

//private function to free automataData stored as a char
void ARC_LexerTokenRule_DestroyCharAutomataDataFn(void *automataData){
    free((char *)automataData);
}

ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharRule(uint32_t id, char character){
    //create the token rule
    ARC_LexerTokenRule tokenRule;

    //set the id
    tokenRule.id = id;

    //create and store the automataData (which is just a char)
    char *automataData = (char *)malloc(sizeof(char));
    *automataData = character;
    tokenRule.automataData = (void *)automataData;

    //we can use the ARC_Lexer_AutomataMatchCharFn for this
    tokenRule.automataFn = ARC_Lexer_AutomataMatchCharFn;

    //add the private destroy function
    tokenRule.destroyAutomataDataFn = ARC_LexerTokenRule_DestroyCharAutomataDataFn;

    //return the created tokenRule
    return tokenRule;
}

//private function to free automataData stored as an ARC_String
void ARC_LexerTokenRule_DestroyStringAutomataDataFn(void *automataData){
    ARC_String_Destroy((ARC_String *)automataData);
}

ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchStringRule(uint32_t id, ARC_String *string){
    //create the token rule
    ARC_LexerTokenRule tokenRule;

    //set the id
    tokenRule.id = id;

    //copy and store the automataData (which is just an ARC_String)
    ARC_String *automataData;
    ARC_String_Copy(&automataData, string);
    tokenRule.automataData = (void *)automataData;

    //we can use the ARC_Lexer_AutomataMatchStringFn for this
    tokenRule.automataFn = ARC_Lexer_AutomataMatchStringFn;

    //add the private destroy function
    tokenRule.destroyAutomataDataFn = ARC_LexerTokenRule_DestroyStringAutomataDataFn;

    //return the created tokenRule
    return tokenRule;
}

ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule(uint32_t id, ARC_String *string){
    //create the token rule
    ARC_LexerTokenRule tokenRule;

    //set the id
    tokenRule.id = id;

    //copy and store the automataData (which is just an ARC_String)
    ARC_String *automataData;
    ARC_String_Copy(&automataData, string);
    tokenRule.automataData = (void *)automataData;

    //we can use the ARC_Lexer_AutomataMatchCharInStringFn for this
    tokenRule.automataFn = ARC_Lexer_AutomataMatchCharInStringFn;

    //add the private destroy function
    tokenRule.destroyAutomataDataFn = ARC_LexerTokenRule_DestroyStringAutomataDataFn;

    //return the created tokenRule
    return tokenRule;
}

void ARC_Lexer_InitBasicTokenRules(ARC_Lexer *lexer){
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_COLON_ID            , ARC_LEXER_TOKEN_COLON_CHAR            ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_SEMICOLON_ID        , ARC_LEXER_TOKEN_SEMICOLON_CHAR        ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_COMMA_ID            , ARC_LEXER_TOKEN_COMMA_CHAR            ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_PERIOD_ID           , ARC_LEXER_TOKEN_PERIOD_CHAR           ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_FORWARD_SLASH_ID    , ARC_LEXER_TOKEN_FORWARD_SLASH_CHAR    ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_BACK_SLASH_ID       , ARC_LEXER_TOKEN_BACK_SLASH_CHAR       ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_LEFT_PARENTHESIS_ID , ARC_LEXER_TOKEN_LEFT_PARENTHESIS_CHAR ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_ID, ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_CHAR));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_ID , ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_CHAR ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_ID, ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_CHAR));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_BANG_ID             , ARC_LEXER_TOKEN_BANG_CHAR             ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_AT_ID               , ARC_LEXER_TOKEN_AT_CHAR               ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_HASH_ID             , ARC_LEXER_TOKEN_HASH_CHAR             ));
    ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_PERCENT_ID          , ARC_LEXER_TOKEN_PERCENT_CHAR          ));
}