herbglitch/archeus
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

older lexer stuff

Browse source
This commit is contained in:
herbglitch 2024-10-16 17:35:38 -06:00
parent 7bd7cc4aa5
commit 380e74a0e6
3 changed files with 211 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

72
include/arc/std/lexer.h
View file

@ -5,7 +5,6 @@
extern "C" {
#endif
#include "arc/std/bool.h"
#include "arc/std/string.h"
#include <stdint.h>
@ -25,16 +24,15 @@ typedef struct ARC_LexerToken {
/**
* @brief checks to see if a string is a type of token
*
* @note do not set tokenData if this function returns ARC_False, doing so will create a memory leak
* @note do not set tokenData if this function returns 0, doing so will create a memory leak
*
* @param[in/out] string a string to be checked to see if it matches a token,
* this needs to srip the token out for the lexer to avoid an infinite loop
* @param[out] tokenData a place to store token data (like a variable name), can be NULL if not needed
* @param[in] automataData any data that needs to be used for the ARC_Lexer_AutomataFn
* @param[out] tokenData a place to store token data (like a variable name), can be NULL if not needed
* @param[in] string a string to be checked to see if it matches a token
* @param[in] automataData any data that needs to be used for the ARC_Lexer_AutomataFn
*
* @return if a token was successfully found ARC_True, otherwise ARC_False
* @return the size of the token found, or 0 if the token was not found
*/
typedef ARC_Bool (* ARC_LexerTokenRule_AutomataFn)(ARC_String **string, ARC_String **tokenData, void *automataData);
typedef uint32_t (* ARC_LexerTokenRule_AutomataFn)(ARC_String **tokenData, ARC_String *string, void *automataData);
/**
* @brief a callback function to clean up ARC_LexerTokenRule's automataData
@ -124,14 +122,13 @@ uint32_t ARC_Lexer_GetTokensSize(ARC_Lexer *lexer);
* @note this is intended as a helper callback
* @note this function is a ARC_Lexer_AutomataFn callback
*
* @param[in/out] string a string to be checked to see if it matches a token,
* this needs to srip the token out for the lexer to avoid an infinite loop
* @param[out] tokenData a place to store token data (like a variable name), can be NULL if not needed
* @param[in] automataData any data that needs to be used for the ARC_Lexer_AutomataFn
* @param[out] tokenData a place to store token data (like a variable name), can be NULL if not needed
* @param[in] string a string to be checked to see if it matches a token
* @param[in] automataData any data that needs to be used for the ARC_Lexer_AutomataFn
*
* @return if a token was successfully found ARC_True, otherwise ARC_False
* @return the size of the token found, or 0 if the token was not found
*/
ARC_Bool ARC_Lexer_AutomataMatchCharFn(ARC_String **string, ARC_String **tokenData, void *automataData);
uint32_t ARC_Lexer_AutomataMatchCharFn(ARC_String **tokenData, ARC_String *string, void *automataData);
/**
* @brief checks if the substring automataData as an ARC_String matches the first part of string
@ -139,14 +136,27 @@ ARC_Bool ARC_Lexer_AutomataMatchCharFn(ARC_String **string, ARC_String **tokenDa
* @note this is intended as a helper callback
* @note this function is a ARC_Lexer_AutomataFn callback
*
* @param[in/out] string a string to be checked to see if it matches a token,
* this needs to srip the token out for the lexer to avoid an infinite loop
* @param[out] tokenData a place to store token data (like a variable name), can be NULL if not needed
* @param[in] automataData any data that needs to be used for the ARC_Lexer_AutomataFn
* @param[out] tokenData a place to store token data (like a variable name), can be NULL if not needed
* @param[in] string a string to be checked to see if it matches a token
* @param[in] automataData any data that needs to be used for the ARC_Lexer_AutomataFn
*
* @return if a token was successfully found ARC_True, otherwise ARC_False
* @return the size of the token found, or 0 if the token was not found
*/
ARC_Bool ARC_Lexer_AutomataMatchStringFn(ARC_String **string, ARC_String **tokenData, void *automataData);
uint32_t ARC_Lexer_AutomataMatchStringFn(ARC_String **tokenData, ARC_String *string, void *automataData);
/**
* @brief checks if the first part of string is a character in substring
*
* @note this is intended as a helper callback
* @note this function is a ARC_Lexer_AutomataFn callback
*
* @param[out] tokenData a place to store token data (like a variable name), can be NULL if not needed
* @param[in] string a string to be checked to see if it matches a token
* @param[in] automataData any data that needs to be used for the ARC_Lexer_AutomataFn
*
* @return the size of the token found, or 0 if the token was not found
*/
uint32_t ARC_Lexer_AutomataMatchCharInStringFn(ARC_String **tokenData, ARC_String *string, void *automataData);
/**
* @brief creates a ARC_LexerTokenRule with a given id and character
@ -173,6 +183,28 @@ ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharRule(uint32_t id,
*/
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchStringRule(uint32_t id, ARC_String *string);
/**
* @brief creates a ARC_LexerTokenRule with a given id and string
*
* @note this is intended as a helper funtion
* #note string will not be freed (it will be copied and the copy will be freed)
*
* @param[in] id a tokens id (basically the token value)
* @param[in] character the string to match against, will be copied
*
* @return a token rule based in the id and string
*/
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule(uint32_t id, ARC_String *string);
/**
* @brief basic tokens
*/
#define ARC_LEXER_TOKEN_NULL 0
#define ARC_LEXER_TOKEN_EOF 1
#define ARC_LEXER_TOKEN_NUMBER 2
#define ARC_LEXER_TOKEN_ALPHACHAR 3
#define ARC_LEXER_TOKEN_WHITESPACE 4
/**
* @brief basic token type ids, chars, and tags
*/