lexer.h

Go to the documentation of this file.

#ifndef ARC_STD_LEXER_H_
#define ARC_STD_LEXER_H_
 
#ifdef __cplusplus
extern "C" {
#endif
#include "arc/std/string.h"
#include <stdint.h>
 
/**
 * @brief a lexer type
*/
typedef struct ARC_Lexer ARC_Lexer;
 
/**
 * @brief a lexer token type
*/
typedef struct ARC_LexerToken {
    uint32_t rule;
    ARC_String *data;
} ARC_LexerToken;
 
/**
 * @brief checks to see if a string is a type of token
 *
 * @note do not set tokenData if this function returns 0, doing so will create a memory leak
 *
 * @param[out] tokenData    a place to store token data (like a variable name), can be NULL if not needed
 * @param[in]  string       a string to be checked to see if it matches a token
 * @param[in]  automataData any data that needs to be used for the ARC_Lexer_AutomataFn
 *
 * @return the size of the token found, or 0 if the token was not found
*/
typedef uint32_t (* ARC_LexerTokenRule_AutomataFn)(ARC_String **tokenData, ARC_String *string, void *automataData);
 
/**
 * @brief a callback function to clean up ARC_LexerTokenRule's automataData
 *
 * @param automataData the void * automataData to destroy
*/
typedef void (* ARC_LexerTokenRule_DestroyAutomataDataFn)(void *automataData);
 
/**
 * @brief a lexer token rule type
*/
typedef struct ARC_LexerTokenRule {
    uint32_t id;
 
    void *automataData;
 
    ARC_LexerTokenRule_AutomataFn automataFn;
    ARC_LexerTokenRule_DestroyAutomataDataFn destroyAutomataDataFn;
} ARC_LexerTokenRule;
 
/**
 * @brief creates an ARC_Lexer type
 *
 * @param[out] lexer ARC_Lexer to create
*/
void ARC_Lexer_Create(ARC_Lexer **lexer);
 
/**
 * @brief destroys an ARC_Lexer type
 *
 * @param[in] lexer ARC_Lexer to free
*/
void ARC_Lexer_Destroy(ARC_Lexer *lexer);
 
/**
 * @brief adds a token rule to a lexer
 *
 * @param [in] lexer     the lexer to add a token rule to
 * @param [in] tokenRule the token rule to add
*/
void ARC_Lexer_RegisterTokenRule(ARC_Lexer *lexer, ARC_LexerTokenRule tokenRule);
 
/**
 * @brief clears all tokens from a lexer (will not clear token rules)
 *
 * @param lexer the lexer to clear tokens from
*/
void ARC_Lexer_Clear(ARC_Lexer *lexer);
 
/**
 * @brief creates tokens using a given string with ARC_LexerToken rules
 *
 * @param[in]     lexer the lexer to get the ARC_LexerTokens from
 * @param[in/out] data the string to lex, will be freed and set to NULL by the end of this function
*/
void ARC_Lexer_LexString(ARC_Lexer *lexer, ARC_String **data);
 
/**
 * @brief reads in and lexs a file
 *
 * @note this function will call ARC_Lexer_LexString, so it's notes are applicable to this function
 *
 * @param[in] lexer the lexer which holds to rules to use
 * @param[in] path  path of file to read in and lex
*/
void ARC_Lexer_LexFile(ARC_Lexer *lexer, ARC_String *path);
 
/**
 * @brief prints rule id and hex of the function name
 *
 * @note this is mostly used for debugging
 *
 * @param[in] lexer the lexer to print rules from
*/
void ARC_Lexer_PrintTokenRules(ARC_Lexer *lexer);
 
/**
 * @brief gets a token at a given index from a lexer
 *
 * @note unless you have a very good reason, you probably don't want to mess with the tokens string.
 *       that will probably change the token's string inside the lexer
 *
 * @param[in] lexer the lexer to get the token from
 * @param[in] index the index of the token in the lexer to get
 *
 * @return a token at the lexer index on success, otherwise NULL
*/
ARC_LexerToken *ARC_Lexer_GetToken(ARC_Lexer *lexer, uint32_t index);
 
/**
 * @brief gets a token at a given index from a lexer
 *
 * @param[in] lexer the lexer to get the tokens size from
 *
 * @return the size of the token array in a lexer
*/
uint32_t ARC_Lexer_GetTokensSize(ARC_Lexer *lexer);
 
/**
 * @brief returns a boolean based on if a lexers rules are continious
 *
 * @param[in] lexer the lexer to check if its ruls are continious
 *
 * @return ARC_True if the set rules are continious
*/
ARC_Bool ARC_Lexer_IsContinious(ARC_Lexer *lexer);
 
/**
 * @brief returns a boolean based on if a given id is a stored token rule id
 *
 * @param[in] lexer the lexer to check stored token rule ids
 * @param[in] id    the id to check against the token rules
 *
 * @return ARC_True if the id is a rule id, ARC_False otherwise
*/
ARC_Bool ARC_Lexer_IsTokenId(ARC_Lexer *lexer, uint32_t id);
 
/**
 * @brief checks if the first character of string matches the automataData cast as a char
 *
 * @note this is intended as a helper callback
 * @note this function is a ARC_Lexer_AutomataFn callback
 *
 * @param[out] tokenData    a place to store token data (like a variable name), can be NULL if not needed
 * @param[in]  string       a string to be checked to see if it matches a token
 * @param[in]  automataData any data that needs to be used for the ARC_Lexer_AutomataFn
 *
 * @return the size of the token found, or 0 if the token was not found
*/
uint32_t ARC_Lexer_AutomataMatchCharFn(ARC_String **tokenData, ARC_String *string, void *automataData);
 
/**
 * @brief checks if the substring automataData as an ARC_String matches the first part of string
 *
 * @note this is intended as a helper callback
 * @note this function is a ARC_Lexer_AutomataFn callback
 *
 * @param[out] tokenData    a place to store token data (like a variable name), can be NULL if not needed
 * @param[in]  string       a string to be checked to see if it matches a token
 * @param[in]  automataData any data that needs to be used for the ARC_Lexer_AutomataFn
 *
 * @return the size of the token found, or 0 if the token was not found
*/
uint32_t ARC_Lexer_AutomataMatchStringFn(ARC_String **tokenData, ARC_String *string, void *automataData);
 
/**
 * @brief checks if the first part of string is a character in substring
 *
 * @note this is intended as a helper callback
 * @note this function is a ARC_Lexer_AutomataFn callback
 *
 * @param[out] tokenData    a place to store token data (like a variable name), can be NULL if not needed
 * @param[in]  string       a string to be checked to see if it matches a token
 * @param[in]  automataData any data that needs to be used for the ARC_Lexer_AutomataFn
 *
 * @return the size of the token found, or 0 if the token was not found
*/
uint32_t ARC_Lexer_AutomataMatchCharInStringFn(ARC_String **tokenData, ARC_String *string, void *automataData);
 
/**
 * @brief creates a ARC_LexerTokenRule with a given id and character
 *
 * @note this is intended as a helper funtion
 *
 * @param[in] id        a tokens id (basically the token value)
 * @param[in] character the character to match against
 *
 * @return a token rule based in the id and character
*/
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharRule(uint32_t id, char character);
 
/**
 * @brief creates a ARC_LexerTokenRule with a given id and character range
 *
 * @note this is intended as a helper funtion
 *
 * @param[in] id    a tokens id (basically the token value)
 * @param[in] start the minimum character value to match against
 * @param[in] end   the maxamum character value to match against
 *
 * @return a token rule based in the id and character
*/
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween(uint32_t id, char start, char end);
 
/**
 * @brief creates a ARC_LexerTokenRule with a given id and string
 *
 * @note this is intended as a helper funtion
 * #note string will not be freed (it will be copied and the copy will be freed)
 *
 * @param[in] id        a tokens id (basically the token value)
 * @param[in] character the string to match against, will be copied
 *
 * @return a token rule based in the id and string
*/
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchStringRule(uint32_t id, ARC_String *string);
 
/**
 * @brief creates a ARC_LexerTokenRule with a given id and string
 *
 * @note this is intended as a helper funtion
 * #note string will not be freed (it will be copied and the copy will be freed)
 *
 * @param[in] id        a tokens id (basically the token value)
 * @param[in] character the string to match against, will be copied
 *
 * @return a token rule based in the id and string
*/
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule(uint32_t id, ARC_String *string);
 
/**
 * @brief basic tokens
*/
#define ARC_LEXER_TOKEN_NULL             0
#define ARC_LEXER_TOKEN_NUMBER           1
#define ARC_LEXER_TOKEN_ALPHA_LOWER_CHAR 2
#define ARC_LEXER_TOKEN_ALPHA_UPPER_CHAR 3
#define ARC_LEXER_TOKEN_WHITESPACE       4
 
/**
 * @brief basic token type ids, chars, and tags
*/
#define ARC_LEXER_TOKEN_NEWLINE_ID             5
#define ARC_LEXER_TOKEN_NEWLINE_CHAR           '\n'
#define ARC_LEXER_TOKEN_COLON_ID               6
#define ARC_LEXER_TOKEN_COLON_CHAR             ':'
#define ARC_LEXER_TOKEN_COLON_TAG              "COLON"
#define ARC_LEXER_TOKEN_SEMICOLON_ID           7
#define ARC_LEXER_TOKEN_SEMICOLON_CHAR         ';'
#define ARC_LEXER_TOKEN_SEMICOLON_TAG          "SEMICOLON"
#define ARC_LEXER_TOKEN_COMMA_ID               8
#define ARC_LEXER_TOKEN_COMMA_CHAR             ','
#define ARC_LEXER_TOKEN_COMMA_TAG              "COMMA"
#define ARC_LEXER_TOKEN_PERIOD_ID              9
#define ARC_LEXER_TOKEN_PERIOD_CHAR            '.'
#define ARC_LEXER_TOKEN_PERIOD_TAG             "PERIOD"
#define ARC_LEXER_TOKEN_FORWARD_SLASH_ID       10
#define ARC_LEXER_TOKEN_FORWARD_SLASH_CHAR     '/'
#define ARC_LEXER_TOKEN_FORWARD_SLASH_TAG      "FORWARD_SLASH"
#define ARC_LEXER_TOKEN_BACK_SLASH_ID          11
#define ARC_LEXER_TOKEN_BACK_SLASH_CHAR        '\\'
#define ARC_LEXER_TOKEN_BACK_SLASH_TAG         "BACK_SLASH"
#define ARC_LEXER_TOKEN_LEFT_PARENTHESIS_ID    12
#define ARC_LEXER_TOKEN_LEFT_PARENTHESIS_CHAR  '('
#define ARC_LEXER_TOKEN_LEFT_PARENTHESIS_TAG   "LEFT_PARENTHESIS"
#define ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_ID   13
#define ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_CHAR ')'
#define ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_TAG  "RIGHT_PARENTHESIS"
#define ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_ID    14
#define ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_CHAR  '{'
#define ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_TAG   "LEFT_CURLY_BRACE"
#define ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_ID   15
#define ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_CHAR '}'
#define ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_TAG  "RIGHT_CURLY_BRACE"
#define ARC_LEXER_TOKEN_BANG_ID                16
#define ARC_LEXER_TOKEN_BANG_CHAR              '!'
#define ARC_LEXER_TOKEN_BANG_TAG               "BANG"
#define ARC_LEXER_TOKEN_AT_ID                  17
#define ARC_LEXER_TOKEN_AT_CHAR                '!'
#define ARC_LEXER_TOKEN_AT_TAG                 "AT"
#define ARC_LEXER_TOKEN_HASH_ID                18
#define ARC_LEXER_TOKEN_HASH_CHAR              '#'
#define ARC_LEXER_TOKEN_HASH_TAG               "HASH"
#define ARC_LEXER_TOKEN_PERCENT_ID             19
#define ARC_LEXER_TOKEN_PERCENT_CHAR           '%'
#define ARC_LEXER_TOKEN_PERCENT_TAG            "PERCENT"
 
/**
 * @brief adds a bunch of basic token rules (matching the BasicTokens above)
*/
void ARC_Lexer_InitBasicTokenRules(ARC_Lexer *lexer);
 
#ifdef __cplusplus
}
#endif
 
#endif // !ARC_STD_LEXER_H_