Loading...
Searching...
No Matches
lexer.c File Reference
`#include "arc/std/lexer.h"`
`#include "arc/std/bool.h"`
`#include "arc/std/errno.h"`
`#include "arc/std/string.h"`
`#include "arc/std/vector.h"`
`#include "arc/std/io.h"`
`#include <stdlib.h>`

Go to the source code of this file.

## Data Structures

struct | ARC_Lexer

## Functions

ARC_Bool | ARC_LexerTokenRule_VectorCompareDataFn (void *dataA, void *dataB)
void | ARC_LexerTokenRule_VectorDestroyDataFn (void *data)
void | ARC_LexerToken_VectorDestroyDataFn (void *data)
void | ARC_Lexer_Create (ARC_Lexer **lexer)
| creates an ARC_Lexer type
void | ARC_Lexer_Destroy (ARC_Lexer *lexer)
| destroys an ARC_Lexer type
void | ARC_Lexer_RegisterTokenRule (ARC_Lexer *lexer, ARC_LexerTokenRule tokenRule)
| adds a token rule to a lexer
void | ARC_Lexer_Clear (ARC_Lexer *lexer)
| clears all tokens from a lexer (will not clear token rules)
void | ARC_Lexer_LexString (ARC_Lexer *lexer, ARC_String **data)
| creates tokens using a given string with ARC_LexerToken rules
void | ARC_Lexer_LexFile (ARC_Lexer *lexer, ARC_String *path)
| reads in and lexs a file
void | ARC_Lexer_PrintTokenRules (ARC_Lexer *lexer)
| prints rule id and hex of the function name
ARC_LexerToken * | ARC_Lexer_GetToken (ARC_Lexer *lexer, uint32_t index)
| gets a token at a given index from a lexer
uint32_t | ARC_Lexer_GetTokensSize (ARC_Lexer *lexer)
| gets a token at a given index from a lexer
ARC_Bool | ARC_Lexer_IsContinious (ARC_Lexer *lexer)
| returns a boolean based on if a lexers rules are continious
ARC_Bool | ARC_Lexer_IsTokenId (ARC_Lexer *lexer, uint32_t id)
| returns a boolean based on if a given id is a stored token rule id
uint32_t | ARC_Lexer_AutomataMatchCharFn (ARC_String **tokenData, ARC_String *string, void *automataData)
| checks if the first character of string matches the automataData cast as a char
uint32_t | ARC_Lexer_AutomataMatchCharOrBetweenFn (ARC_String **tokenData, ARC_String *string, void *automataData)
uint32_t | ARC_Lexer_AutomataMatchStringFn (ARC_String **tokenData, ARC_String *string, void *automataData)
| checks if the substring automataData as an ARC_String matches the first part of string
uint32_t | ARC_Lexer_AutomataMatchCharInStringFn (ARC_String **tokenData, ARC_String *string, void *automataData)
| checks if the first part of string is a character in substring
void | ARC_LexerTokenRule_DestroyCharAutomataDataFn (void *automataData)
ARC_LexerTokenRule | ARC_LexerTokenRule_CreateAndReturnMatchCharRule (uint32_t id, char character)
| creates a ARC_LexerTokenRule with a given id and character
ARC_LexerTokenRule | ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween (uint32_t id, char start, char end)
| creates a ARC_LexerTokenRule with a given id and character range
void | ARC_LexerTokenRule_DestroyStringAutomataDataFn (void *automataData)
ARC_LexerTokenRule | ARC_LexerTokenRule_CreateAndReturnMatchStringRule (uint32_t id, ARC_String *string)
| creates a ARC_LexerTokenRule with a given id and string
ARC_LexerTokenRule | ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule (uint32_t id, ARC_String *string)
| creates a ARC_LexerTokenRule with a given id and string
void | ARC_Lexer_InitBasicTokenRules (ARC_Lexer *lexer)
| adds a bunch of basic token rules (matching the BasicTokens above)

## Function Documentation

## ARC_Lexer_AutomataMatchCharFn()

uint32_t ARC_Lexer_AutomataMatchCharFn | ( | ARC_String ** | tokenData,
|  | ARC_String * | string,
|  | void * | automataData )

checks if the first character of string matches the automataData cast as a char

Note
  this is intended as a helper callback
  this function is a ARC_Lexer_AutomataFn callback

Parameters

  [out] | tokenData | a place to store token data (like a variable name), can be NULL if not needed
  [in] | string | a string to be checked to see if it matches a token
  [in] | automataData | any data that needs to be used for the ARC_Lexer_AutomataFn

Returns
  the size of the token found, or 0 if the token was not found

Definition at line 311 of file lexer.c.

{
//if there is a match the token will be the same as automataData, so we don't need to store it again
*tokenData = NULL;
//check to see if there is a match with automataData as a char
if(string->data[0] == *(char *)automataData){
//return the token was found of length 1
return 1;
}
//no match was found
return 0;
}
ARC_String::data
char * data
Definition string.h:15

References ARC_String::data.

Referenced by ARC_LexerTokenRule_CreateAndReturnMatchCharRule().

## ARC_Lexer_AutomataMatchCharInStringFn()

uint32_t ARC_Lexer_AutomataMatchCharInStringFn | ( | ARC_String ** | tokenData,
|  | ARC_String * | string,
|  | void * | automataData )

checks if the first part of string is a character in substring

Note
  this is intended as a helper callback
  this function is a ARC_Lexer_AutomataFn callback

Parameters

  [out] | tokenData | a place to store token data (like a variable name), can be NULL if not needed
  [in] | string | a string to be checked to see if it matches a token
  [in] | automataData | any data that needs to be used for the ARC_Lexer_AutomataFn

Returns
  the size of the token found, or 0 if the token was not found

Definition at line 356 of file lexer.c.

{
//if there is a match the token will be the same as automataData, so we don't need to store it again
*tokenData = NULL;
//check to see if there is a char match in automataData as a string
ARC_String *automataDataString = (ARC_String *)automataData;
for(uint64_t index = 0; index < automataDataString->length; index++){
if(string->data[0] == automataDataString->data[index]){
//return the token was found in the string of length 1
return 1;
}
}
//no match was found
return 0;
}
ARC_String
substring position within a string
Definition string.h:14
ARC_String::length
uint64_t length
Definition string.h:16

References ARC_String::data, and ARC_String::length.

Referenced by ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule().

## ARC_Lexer_AutomataMatchCharOrBetweenFn()

uint32_t ARC_Lexer_AutomataMatchCharOrBetweenFn | ( | ARC_String ** | tokenData,
|  | ARC_String * | string,
|  | void * | automataData )

Definition at line 325 of file lexer.c.

{
//if there is a match the token will be the same as automataData, so we don't need to store it again
*tokenData = NULL;
//check to see if there is a match with automataData as a range of chars
char *automataDataChars = (char *)automataData;
if(string->data[0] >= automataDataChars[0] && string->data[0] <= ((char *)automataData)[1]){
//return the token as token data and the token was found of length 1
ARC_String_Create(tokenData, string->data, 1);
return 1;
}
//no match was found
return 0;
}
ARC_String_Create
void ARC_String_Create(ARC_String **string, char *data, uint64_t length)
creates ARC_String type
Definition string.c:9

References ARC_String_Create(), and ARC_String::data.

Referenced by ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween().

## ARC_Lexer_AutomataMatchStringFn()

uint32_t ARC_Lexer_AutomataMatchStringFn | ( | ARC_String ** | tokenData,
|  | ARC_String * | string,
|  | void * | automataData )

checks if the substring automataData as an ARC_String matches the first part of string

Note
  this is intended as a helper callback
  this function is a ARC_Lexer_AutomataFn callback

Parameters

  [out] | tokenData | a place to store token data (like a variable name), can be NULL if not needed
  [in] | string | a string to be checked to see if it matches a token
  [in] | automataData | any data that needs to be used for the ARC_Lexer_AutomataFn

Returns
  the size of the token found, or 0 if the token was not found

Definition at line 341 of file lexer.c.

{
//if there is a match the token will be the same as automataData, so we don't need to store it again
*tokenData = NULL;
//check to see if there is a match with automataData as a string
ARC_String *automataDataString = (ARC_String *)automataData;
if(ARC_String_SubstringEquals(string, 0, automataDataString)){
//return the token was found of the string length
return automataDataString->length;
}
//no match was found
return 0;
}
ARC_String_SubstringEquals
ARC_Bool ARC_String_SubstringEquals(ARC_String *first, uint64_t offset, ARC_String *second)
check if substring of first equals second string
Definition string.c:175

References ARC_String_SubstringEquals(), and ARC_String::length.

Referenced by ARC_LexerTokenRule_CreateAndReturnMatchStringRule().

## ARC_Lexer_Clear()

void ARC_Lexer_Clear | ( | ARC_Lexer * | lexer | ) |

clears all tokens from a lexer (will not clear token rules)

Parameters

  lexer | the lexer to clear tokens from

Definition at line 161 of file lexer.c.

{
//clear the tokens vector
ARC_Vector_Clear(lexer->tokens);
}
ARC_Lexer::tokens
ARC_Vector * tokens
Definition lexer.c:12
ARC_Vector_Clear
void ARC_Vector_Clear(ARC_Vector *vector)
clears all items from a vector
Definition vector.c:142

References ARC_Vector_Clear(), and ARC_Lexer::tokens.

Referenced by ARC_Parser_Parse().

## ARC_Lexer_Create()

void ARC_Lexer_Create | ( | ARC_Lexer ** | lexer | ) |

creates an ARC_Lexer type

Parameters

  [out] | lexer | ARC_Lexer to create

Definition at line 50 of file lexer.c.

{
//create the lexer
*lexer = (ARC_Lexer *)malloc(sizeof(ARC_Lexer));
//setup token rules vector with compare and delete functions
ARC_Vector_CompareDataFn tokenRulesVectorCompareDataFn = ARC_LexerTokenRule_VectorCompareDataFn;
ARC_Vector_DestroyDataFn tokenRulesVectorDestroyDataFn = ARC_LexerTokenRule_VectorDestroyDataFn;
ARC_Vector_Create(&(*lexer)->tokenRules, &tokenRulesVectorCompareDataFn, &tokenRulesVectorDestroyDataFn);
//setup tokens vector with delete funtion, we don't want a deleteDataFn because their index will be used as the id
ARC_Vector_DestroyDataFn tokenVectorDestroyDataFn = ARC_LexerToken_VectorDestroyDataFn;
ARC_Vector_Create(&(*lexer)->tokens, NULL, &tokenVectorDestroyDataFn);
//set token rules to continuous and initialize the token rules max value
(*lexer)->tokenRulesAreContinuous = ARC_True;
(*lexer)->tokenRulesMaxVal = 0;
}
ARC_True
#define ARC_True
Definition bool.h:11
ARC_LexerTokenRule_VectorCompareDataFn
ARC_Bool ARC_LexerTokenRule_VectorCompareDataFn(void *dataA, void *dataB)
Definition lexer.c:20
ARC_LexerTokenRule_VectorDestroyDataFn
void ARC_LexerTokenRule_VectorDestroyDataFn(void *data)
Definition lexer.c:32
ARC_LexerToken_VectorDestroyDataFn
void ARC_LexerToken_VectorDestroyDataFn(void *data)
Definition lexer.c:39
ARC_Lexer
Definition lexer.c:10
ARC_Vector_DestroyDataFn
void(* ARC_Vector_DestroyDataFn)(void *data)
a callback that cleans up memory when it is removed from the vector
Definition vector.h:31
ARC_Vector_CompareDataFn
ARC_Bool(* ARC_Vector_CompareDataFn)(void *dataA, void *dataB)
a callback that allows the user to define a way to check the data stored in a vector for a match
Definition vector.h:24
ARC_Vector_Create
void ARC_Vector_Create(ARC_Vector **vector, ARC_Vector_CompareDataFn *compareDataFn, ARC_Vector_DestroyDataFn *destroyDataFn)
creates an ARC_Vector which is an "expandable" array
Definition vector.c:31

References ARC_LexerToken_VectorDestroyDataFn(), ARC_LexerTokenRule_VectorCompareDataFn(), ARC_LexerTokenRule_VectorDestroyDataFn(), ARC_True, and ARC_Vector_Create().

Referenced by ARC_Parser_Create(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), and ARC_TEST().

## ARC_Lexer_Destroy()

void ARC_Lexer_Destroy | ( | ARC_Lexer * | lexer | ) |

destroys an ARC_Lexer type

Parameters

  [in] | lexer | ARC_Lexer to free

Definition at line 68 of file lexer.c.

{
//free the tokens (there is a vectorDeleteDataFn, so tokens should be freed)
ARC_Vector_Destroy(lexer->tokens);
//free the token rules (there is a vectorDeleteDataFn, so token rules should be freed)
ARC_Vector_Destroy(lexer->tokenRules);
//free the lexer
free(lexer);
}
ARC_Lexer::tokenRules
ARC_Vector * tokenRules
Definition lexer.c:11
ARC_Vector_Destroy
void ARC_Vector_Destroy(ARC_Vector *vector)
destroys an ARC_Vector
Definition vector.c:54

References ARC_Vector_Destroy(), ARC_Lexer::tokenRules, and ARC_Lexer::tokens.

Referenced by ARC_Parser_Destroy(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), and ARC_TEST().

## ARC_Lexer_GetToken()

ARC_LexerToken * ARC_Lexer_GetToken | ( | ARC_Lexer * | lexer,
|  | uint32_t | index )

gets a token at a given index from a lexer

Note
  unless you have a very good reason, you probably don't want to mess with the tokens string. that will probably change the token's string inside the lexer

Parameters

  [in] | lexer | the lexer to get the token from
  [in] | index | the index of the token in the lexer to get

Returns
  a token at the lexer index on success, otherwise NULL

Definition at line 272 of file lexer.c.

{
//get the token and log if there is an error
ARC_LexerToken *token = ARC_Vector_Get(lexer->tokens, index);
if(arc_errno){
ARC_DEBUG_LOG_ERROR("ARC_Lexer_GetToken(lexer, index), errored when running ARC_Vector_Get(lexer->tokens, index);. check logs for more info");
//return a token with max rule value, and NULL for the string to signify an error
return NULL;
}
//the token was found, so return it
return token;
}
arc_errno
int32_t arc_errno
Definition errno.c:5
ARC_DEBUG_LOG_ERROR
#define ARC_DEBUG_LOG_ERROR(STR)
Definition errno.h:39
ARC_LexerToken
a lexer token type
Definition lexer.h:18
ARC_Vector_Get
void * ARC_Vector_Get(ARC_Vector *vector, uint32_t index)
gets an item from an ARC_Vector at a position index
Definition vector.c:153

References ARC_DEBUG_LOG_ERROR, arc_errno, ARC_Vector_Get(), and ARC_Lexer::tokens.

Referenced by ARC_Parser_ParseTag(), and ARC_TEST().

## ARC_Lexer_GetTokensSize()

uint32_t ARC_Lexer_GetTokensSize | ( | ARC_Lexer * | lexer | ) |

gets a token at a given index from a lexer

Parameters

  [in] | lexer | the lexer to get the tokens size from

Returns
  the size of the token array in a lexer

Definition at line 286 of file lexer.c.

{
return ARC_Vector_GetSize(lexer->tokens);
}
ARC_Vector_GetSize
uint32_t ARC_Vector_GetSize(ARC_Vector *vector)
gets the current size of an ARC_Vector as an unsigned 32 bit integer
Definition vector.c:149

References ARC_Vector_GetSize(), and ARC_Lexer::tokens.

Referenced by ARC_Parser_Parse(), and ARC_Parser_ParseTag().

## ARC_Lexer_InitBasicTokenRules()

void ARC_Lexer_InitBasicTokenRules | ( | ARC_Lexer * | lexer | ) |

adds a bunch of basic token rules (matching the BasicTokens above)

Definition at line 472 of file lexer.c.

{
//null
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_NULL, 0));
//number
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween(ARC_LEXER_TOKEN_NUMBER, '0', '9'));
//alpha char
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween(ARC_LEXER_TOKEN_ALPHA_LOWER_CHAR, 'a', 'z'));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween(ARC_LEXER_TOKEN_ALPHA_UPPER_CHAR, 'A', 'Z'));
//whitespace
//TODO: fix this
ARC_String *whitespaceString;
ARC_String_CreateWithStrlen(&whitespaceString, " \t");
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule(ARC_LEXER_TOKEN_WHITESPACE, whitespaceString));
ARC_String_Destroy(whitespaceString);
//single char tokens
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_NEWLINE_ID , ARC_LEXER_TOKEN_NEWLINE_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_COLON_ID , ARC_LEXER_TOKEN_COLON_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_SEMICOLON_ID , ARC_LEXER_TOKEN_SEMICOLON_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_COMMA_ID , ARC_LEXER_TOKEN_COMMA_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_PERIOD_ID , ARC_LEXER_TOKEN_PERIOD_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_FORWARD_SLASH_ID , ARC_LEXER_TOKEN_FORWARD_SLASH_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_BACK_SLASH_ID , ARC_LEXER_TOKEN_BACK_SLASH_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_LEFT_PARENTHESIS_ID , ARC_LEXER_TOKEN_LEFT_PARENTHESIS_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_ID, ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_CHAR));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_ID , ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_ID, ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_CHAR));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_BANG_ID , ARC_LEXER_TOKEN_BANG_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_AT_ID , ARC_LEXER_TOKEN_AT_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_HASH_ID , ARC_LEXER_TOKEN_HASH_CHAR ));
ARC_Lexer_RegisterTokenRule(lexer, ARC_LexerTokenRule_CreateAndReturnMatchCharRule(ARC_LEXER_TOKEN_PERCENT_ID , ARC_LEXER_TOKEN_PERCENT_CHAR ));
}
ARC_LEXER_TOKEN_WHITESPACE
#define ARC_LEXER_TOKEN_WHITESPACE
Definition lexer.h:252
ARC_LEXER_TOKEN_FORWARD_SLASH_ID
#define ARC_LEXER_TOKEN_FORWARD_SLASH_ID
Definition lexer.h:271
ARC_LEXER_TOKEN_ALPHA_UPPER_CHAR
#define ARC_LEXER_TOKEN_ALPHA_UPPER_CHAR
Definition lexer.h:251
ARC_LEXER_TOKEN_PERIOD_CHAR
#define ARC_LEXER_TOKEN_PERIOD_CHAR
Definition lexer.h:269
ARC_LEXER_TOKEN_PERCENT_ID
#define ARC_LEXER_TOKEN_PERCENT_ID
Definition lexer.h:298
ARC_LEXER_TOKEN_PERCENT_CHAR
#define ARC_LEXER_TOKEN_PERCENT_CHAR
Definition lexer.h:299
ARC_LEXER_TOKEN_LEFT_PARENTHESIS_CHAR
#define ARC_LEXER_TOKEN_LEFT_PARENTHESIS_CHAR
Definition lexer.h:278
ARC_LEXER_TOKEN_PERIOD_ID
#define ARC_LEXER_TOKEN_PERIOD_ID
Definition lexer.h:268
ARC_LEXER_TOKEN_NULL
#define ARC_LEXER_TOKEN_NULL
basic tokens
Definition lexer.h:248
ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_CHAR
#define ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_CHAR
Definition lexer.h:287
ARC_LEXER_TOKEN_COMMA_CHAR
#define ARC_LEXER_TOKEN_COMMA_CHAR
Definition lexer.h:266
ARC_LEXER_TOKEN_AT_ID
#define ARC_LEXER_TOKEN_AT_ID
Definition lexer.h:292
ARC_LEXER_TOKEN_AT_CHAR
#define ARC_LEXER_TOKEN_AT_CHAR
Definition lexer.h:293
ARC_LEXER_TOKEN_COLON_CHAR
#define ARC_LEXER_TOKEN_COLON_CHAR
Definition lexer.h:260
ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_ID
#define ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_ID
Definition lexer.h:283
ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_ID
#define ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_ID
Definition lexer.h:280
ARC_LEXER_TOKEN_BANG_CHAR
#define ARC_LEXER_TOKEN_BANG_CHAR
Definition lexer.h:290
ARC_LEXER_TOKEN_NEWLINE_ID
#define ARC_LEXER_TOKEN_NEWLINE_ID
basic token type ids, chars, and tags
Definition lexer.h:257
ARC_LEXER_TOKEN_COMMA_ID
#define ARC_LEXER_TOKEN_COMMA_ID
Definition lexer.h:265
ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_ID
#define ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_ID
Definition lexer.h:286
ARC_LEXER_TOKEN_HASH_ID
#define ARC_LEXER_TOKEN_HASH_ID
Definition lexer.h:295
ARC_LEXER_TOKEN_NEWLINE_CHAR
#define ARC_LEXER_TOKEN_NEWLINE_CHAR
Definition lexer.h:258
ARC_LEXER_TOKEN_BACK_SLASH_ID
#define ARC_LEXER_TOKEN_BACK_SLASH_ID
Definition lexer.h:274
ARC_LEXER_TOKEN_BANG_ID
#define ARC_LEXER_TOKEN_BANG_ID
Definition lexer.h:289
ARC_LEXER_TOKEN_SEMICOLON_ID
#define ARC_LEXER_TOKEN_SEMICOLON_ID
Definition lexer.h:262
ARC_LEXER_TOKEN_FORWARD_SLASH_CHAR
#define ARC_LEXER_TOKEN_FORWARD_SLASH_CHAR
Definition lexer.h:272
ARC_LEXER_TOKEN_ALPHA_LOWER_CHAR
#define ARC_LEXER_TOKEN_ALPHA_LOWER_CHAR
Definition lexer.h:250
ARC_LEXER_TOKEN_BACK_SLASH_CHAR
#define ARC_LEXER_TOKEN_BACK_SLASH_CHAR
Definition lexer.h:275
ARC_LEXER_TOKEN_COLON_ID
#define ARC_LEXER_TOKEN_COLON_ID
Definition lexer.h:259
ARC_LEXER_TOKEN_SEMICOLON_CHAR
#define ARC_LEXER_TOKEN_SEMICOLON_CHAR
Definition lexer.h:263
ARC_LEXER_TOKEN_LEFT_PARENTHESIS_ID
#define ARC_LEXER_TOKEN_LEFT_PARENTHESIS_ID
Definition lexer.h:277
ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_CHAR
#define ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_CHAR
Definition lexer.h:284
ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_CHAR
#define ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_CHAR
Definition lexer.h:281
ARC_LEXER_TOKEN_NUMBER
#define ARC_LEXER_TOKEN_NUMBER
Definition lexer.h:249
ARC_LEXER_TOKEN_HASH_CHAR
#define ARC_LEXER_TOKEN_HASH_CHAR
Definition lexer.h:296
ARC_Lexer_RegisterTokenRule
void ARC_Lexer_RegisterTokenRule(ARC_Lexer *lexer, ARC_LexerTokenRule tokenRule)
adds a token rule to a lexer
Definition lexer.c:79
ARC_LexerTokenRule_CreateAndReturnMatchCharRule
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharRule(uint32_t id, char character)
creates a ARC_LexerTokenRule with a given id and character
Definition lexer.c:378
ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween(uint32_t id, char start, char end)
creates a ARC_LexerTokenRule with a given id and character range
Definition lexer.c:400
ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule
ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule(uint32_t id, ARC_String *string)
creates a ARC_LexerTokenRule with a given id and string
Definition lexer.c:450
ARC_String_CreateWithStrlen
void ARC_String_CreateWithStrlen(ARC_String **string, char *data)
creates ARC_String type with strinlen
Definition string.c:32
ARC_String_Destroy
void ARC_String_Destroy(ARC_String *string)
destroys ARC_String type
Definition string.c:52

References ARC_Lexer_RegisterTokenRule(), ARC_LEXER_TOKEN_ALPHA_LOWER_CHAR, ARC_LEXER_TOKEN_ALPHA_UPPER_CHAR, ARC_LEXER_TOKEN_AT_CHAR, ARC_LEXER_TOKEN_AT_ID, ARC_LEXER_TOKEN_BACK_SLASH_CHAR, ARC_LEXER_TOKEN_BACK_SLASH_ID, ARC_LEXER_TOKEN_BANG_CHAR, ARC_LEXER_TOKEN_BANG_ID, ARC_LEXER_TOKEN_COLON_CHAR, ARC_LEXER_TOKEN_COLON_ID, ARC_LEXER_TOKEN_COMMA_CHAR, ARC_LEXER_TOKEN_COMMA_ID, ARC_LEXER_TOKEN_FORWARD_SLASH_CHAR, ARC_LEXER_TOKEN_FORWARD_SLASH_ID, ARC_LEXER_TOKEN_HASH_CHAR, ARC_LEXER_TOKEN_HASH_ID, ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_CHAR, ARC_LEXER_TOKEN_LEFT_CURLY_BRACE_ID, ARC_LEXER_TOKEN_LEFT_PARENTHESIS_CHAR, ARC_LEXER_TOKEN_LEFT_PARENTHESIS_ID, ARC_LEXER_TOKEN_NEWLINE_CHAR, ARC_LEXER_TOKEN_NEWLINE_ID, ARC_LEXER_TOKEN_NULL, ARC_LEXER_TOKEN_NUMBER, ARC_LEXER_TOKEN_PERCENT_CHAR, ARC_LEXER_TOKEN_PERCENT_ID, ARC_LEXER_TOKEN_PERIOD_CHAR, ARC_LEXER_TOKEN_PERIOD_ID, ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_CHAR, ARC_LEXER_TOKEN_RIGHT_CURLY_BRACE_ID, ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_CHAR, ARC_LEXER_TOKEN_RIGHT_PARENTHESIS_ID, ARC_LEXER_TOKEN_SEMICOLON_CHAR, ARC_LEXER_TOKEN_SEMICOLON_ID, ARC_LEXER_TOKEN_WHITESPACE, ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule(), ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween(), ARC_LexerTokenRule_CreateAndReturnMatchCharRule(), ARC_String_CreateWithStrlen(), and ARC_String_Destroy().

Referenced by ARC_TEST(), and TEST_Parser_InitLexerRulesFn().

## ARC_Lexer_IsContinious()

ARC_Bool ARC_Lexer_IsContinious | ( | ARC_Lexer * | lexer | ) |

returns a boolean based on if a lexers rules are continious

Parameters

  [in] | lexer | the lexer to check if its ruls are continious

Returns
  ARC_True if the set rules are continious

Definition at line 290 of file lexer.c.

{
return lexer->tokenRulesAreContinuous;
}
ARC_Lexer::tokenRulesAreContinuous
ARC_Bool tokenRulesAreContinuous
Definition lexer.c:15

References ARC_Lexer::tokenRulesAreContinuous.

Referenced by ARC_TEST(), and ARC_TEST().

## ARC_Lexer_IsTokenId()

ARC_Bool ARC_Lexer_IsTokenId | ( | ARC_Lexer * | lexer,
|  | uint32_t | id )

returns a boolean based on if a given id is a stored token rule id

Parameters

  [in] | lexer | the lexer to check stored token rule ids
  [in] | id | the id to check against the token rules

Returns
  ARC_True if the id is a rule id, ARC_False otherwise

Definition at line 294 of file lexer.c.

{
//if the rules are continuous we can just check if it is less than the max rules value
if(lexer->tokenRulesAreContinuous == ARC_True){
return id <= lexer->tokenRulesMaxVal;
}
//the rules are not continuous so we need to check each individually
for(uint32_t index = 0; index < ARC_Vector_GetSize(lexer->tokenRules); index++){
ARC_LexerTokenRule *currentTokenRule = (ARC_LexerTokenRule *)ARC_Vector_Get(lexer->tokenRules, index);
if(currentTokenRule->id == id){
return ARC_True;
}
}
return ARC_False;
}
ARC_False
#define ARC_False
Definition bool.h:12
ARC_LexerTokenRule
a lexer token rule type
Definition lexer.h:46
ARC_LexerTokenRule::id
uint32_t id
Definition lexer.h:47
ARC_Lexer::tokenRulesMaxVal
uint32_t tokenRulesMaxVal
Definition lexer.c:16

References ARC_False, ARC_True, ARC_Vector_Get(), ARC_Vector_GetSize(), ARC_LexerTokenRule::id, ARC_Lexer::tokenRules, ARC_Lexer::tokenRulesAreContinuous, and ARC_Lexer::tokenRulesMaxVal.

Referenced by ARC_Parser_ParseTag(), ARC_TEST(), ARC_TEST(), and ARC_TEST().

## ARC_Lexer_LexFile()

void ARC_Lexer_LexFile | ( | ARC_Lexer * | lexer,
|  | ARC_String * | path )

reads in and lexs a file

Note
  this function will call ARC_Lexer_LexString, so it's notes are applicable to this function

Parameters

  [in] | lexer | the lexer which holds to rules to use
  [in] | path | path of file to read in and lex

Definition at line 245 of file lexer.c.

{
//read file and clean up if it errors
ARC_String *data;
ARC_IO_FileToStr(path, &data);
if(arc_errno){
ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexFile(lexer, path), errored when running ARC_IO_FileToStr(path, &data);. check logs for more info");
if(data != NULL){
ARC_String_Destroy(data);
}
return;
}
//lex the string and log if there is an error, ARC_Lexer_LexString will clean up the string
ARC_Lexer_LexString(lexer, &data);
if(arc_errno){
ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexFile(lexer, path), errored when running ARC_Lexer_LexString(lexer, data);. check logs for more info");
}
}
ARC_IO_FileToStr
void ARC_IO_FileToStr(ARC_String *path, ARC_String **data)
get string and size from file
Definition io.c:43
ARC_Lexer_LexString
void ARC_Lexer_LexString(ARC_Lexer *lexer, ARC_String **data)
creates tokens using a given string with ARC_LexerToken rules
Definition lexer.c:166

References ARC_DEBUG_LOG_ERROR, arc_errno, ARC_IO_FileToStr(), ARC_Lexer_LexString(), and ARC_String_Destroy().

## ARC_Lexer_LexString()

void ARC_Lexer_LexString | ( | ARC_Lexer * | lexer,
|  | ARC_String ** | data )

creates tokens using a given string with ARC_LexerToken rules

Parameters

  [in] | lexer | the lexer to get the ARC_LexerTokens from
  | [in/out] | data the string to lex, will be freed and set to NULL by the end of this function

Definition at line 166 of file lexer.c.

{
//check if there are any token rules to use
if(ARC_Vector_GetSize(lexer->tokenRules) == 0){
arc_errno = ARC_ERRNO_DATA;
ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexString(lexer, data), no tokens registered to lexer to use");
return;
}
//this will run untill everything token is stripped or there is an error
while(*data != NULL){
uint32_t tokenLength = 0;
uint32_t lastTokenLength = 0;
ARC_LexerToken *token = NULL;
for(uint32_t index = 0; index < ARC_Vector_GetSize(lexer->tokenRules); index++){
//check if the token rule is found
ARC_LexerTokenRule *tokenRule = ARC_Vector_Get(lexer->tokenRules, index);
//tokenData should only exist if tokenLength is ARC_True as stated in the header
ARC_String *tokenData = NULL;
tokenLength = tokenRule->automataFn(&tokenData, *data, tokenRule->automataData);
//check if a token was found if it wasn't continue. I'm doing this to try to cut down on the ammount of indentation
if(tokenLength == 0){
continue;
}
//check to see if we found a better match
if(tokenLength > lastTokenLength){
//free the current token if it exists
if(token != NULL){
ARC_LexerToken_VectorDestroyDataFn((void *)token);
}
//create the token to add
token = (ARC_LexerToken *)malloc(sizeof(ARC_LexerToken));
token->rule = tokenRule->id;
token->data = tokenData;
//update the last found tokenLength to the max length
lastTokenLength = tokenLength;
}
}
//if no token was found, throw an error
if(token == NULL){
arc_errno = ARC_ERRNO_DATA;
ARC_DEBUG_LOG_ERROR_WITH_VARIABLES("ARC_Lexer_LexString(lexer, data), no tokens found with current string: \"%s\"", (*data)->data);
//clean up errored string
ARC_String_Destroy(*data);
*data = NULL;
return;
}
//token exists (something must have gone very wrong if it doesn't), so add it and check for overflow (which I'd be surprised if that happens)
ARC_Vector_Add(lexer->tokens, (void *)token);
if(arc_errno){
ARC_DEBUG_LOG_ERROR("ARC_Lexer_LexString(lexer, data), errored when running ARC_Vector_Add(lexer->tokens, token);. check logs for more info");
free(token);
//clean up errored string
ARC_String_Destroy(*data);
*data = NULL;
return;
}
//if the last token was found, destroy the string and return
if(lastTokenLength == (*data)->length){
ARC_String_Destroy(*data);
*data = NULL;
return;
}
//strip the string
ARC_String_ReplaceWithSubstring(data, lastTokenLength, (*data)->length - lastTokenLength);
}
}
ARC_DEBUG_LOG_ERROR_WITH_VARIABLES
#define ARC_DEBUG_LOG_ERROR_WITH_VARIABLES(STR,...)
Definition errno.h:40
ARC_ERRNO_DATA
#define ARC_ERRNO_DATA
Definition errno.h:7
ARC_String_ReplaceWithSubstring
void ARC_String_ReplaceWithSubstring(ARC_String **string, uint64_t start, uint64_t length)
replaces a string with a section of itself
Definition string.c:100
ARC_LexerTokenRule::automataData
void * automataData
Definition lexer.h:49
ARC_LexerTokenRule::automataFn
ARC_LexerTokenRule_AutomataFn automataFn
Definition lexer.h:51
ARC_LexerToken::rule
uint32_t rule
Definition lexer.h:19
ARC_LexerToken::data
ARC_String * data
Definition lexer.h:20
ARC_Vector_Add
void ARC_Vector_Add(ARC_Vector *vector, void *data)
adds an item to an ARC_Vector
Definition vector.c:70

References ARC_DEBUG_LOG_ERROR, ARC_DEBUG_LOG_ERROR_WITH_VARIABLES, arc_errno, ARC_ERRNO_DATA, ARC_LexerToken_VectorDestroyDataFn(), ARC_String_Destroy(), ARC_String_ReplaceWithSubstring(), ARC_Vector_Add(), ARC_Vector_Get(), ARC_Vector_GetSize(), ARC_LexerTokenRule::automataData, ARC_LexerTokenRule::automataFn, ARC_LexerToken::data, ARC_LexerTokenRule::id, ARC_String::length, ARC_LexerToken::rule, ARC_Lexer::tokenRules, and ARC_Lexer::tokens.

Referenced by ARC_Lexer_LexFile(), ARC_Parser_Parse(), and ARC_TEST().

## ARC_Lexer_PrintTokenRules()

void ARC_Lexer_PrintTokenRules | ( | ARC_Lexer * | lexer | ) |

prints rule id and hex of the function name

Note
  this is mostly used for debugging

Parameters

  [in] | lexer | the lexer to print rules from

Definition at line 265 of file lexer.c.

{
for(uint32_t index = 0; index < ARC_Vector_GetSize(lexer->tokenRules); index++){
ARC_LexerTokenRule *tokenRule = ARC_Vector_Get(lexer->tokenRules, index);
printf("Rule: %02i\tFunction: %p\n", tokenRule->id, tokenRule->automataFn);
}
}

References ARC_Vector_Get(), ARC_Vector_GetSize(), ARC_LexerTokenRule::automataFn, ARC_LexerTokenRule::id, and ARC_Lexer::tokenRules.

## ARC_Lexer_RegisterTokenRule()

void ARC_Lexer_RegisterTokenRule | ( | ARC_Lexer * | lexer,
|  | ARC_LexerTokenRule | tokenRule )

adds a token rule to a lexer

Parameters

  [in] | lexer | the lexer to add a token rule to
  [in] | tokenRule | the token rule to add

Definition at line 79 of file lexer.c.

{
//copy the token to a new pointer
ARC_LexerTokenRule *storedTokenRule = (ARC_LexerTokenRule *)malloc(sizeof(ARC_LexerTokenRule));
*storedTokenRule = tokenRule;
//TODO: add warning here for if arc_errno is already set
//add to the vector and check for error (I'd be surprised if the error ever happened because that would most likely mean overflow)
ARC_Vector_Add(lexer->tokenRules, storedTokenRule);
if(arc_errno){
ARC_DEBUG_LOG_ERROR("ARC_Lexer_RegisterTokenRule(lexer, tokenRule), errored when running ARC_Vector_Add(lexer->tokenRules, storedTokenRule);. check logs for more info");
free(storedTokenRule);
}
//check if the value still is continuous
if(lexer->tokenRulesAreContinuous == ARC_True){
//if it is already continuous we just check if it is one value above the tokens already in the vector
for(uint32_t tokenRuleIndex = ARC_Vector_GetSize(lexer->tokenRules) - 1; tokenRuleIndex > 0; tokenRuleIndex--){
//get the current token rule
ARC_LexerTokenRule *currentTokenRule = (ARC_LexerTokenRule *)ARC_Vector_Get(lexer->tokenRules, tokenRuleIndex - 1);
//check if the token rule is continuous (then next max value by one)
if(tokenRule.id - currentTokenRule->id == 1){
//the token rule is already continuous so we can update the max value and return
lexer->tokenRulesMaxVal = tokenRule.id;
return;
}
}
//the token is no longer continous
lexer->tokenRulesAreContinuous = ARC_False;
return;
}
//check to see if this value makes the token rule continuous again
//TODO: might want to optomize this
uint32_t minValue = ~(uint32_t)0;
for(uint32_t tokenRuleIndex = 0; tokenRuleIndex < ARC_Vector_GetSize(lexer->tokenRules); tokenRuleIndex++){
//get the current token rule
ARC_LexerTokenRule *currentTokenRule = (ARC_LexerTokenRule *)ARC_Vector_Get(lexer->tokenRules, tokenRuleIndex);
//check each token to find the minimum one
if(currentTokenRule->id < minValue){
minValue = currentTokenRule->id;
}
}
//loop through untill either all the values are checked and in order or the token rule is not continuous
//TODO: might want to optomize this
for(uint32_t foundSize = 0; foundSize != ARC_Vector_GetSize(lexer->tokenRules); foundSize++){
//check all current rules
ARC_Bool currentAreContinuous = ARC_False;
for(uint32_t tokenRuleIndex = 0; tokenRuleIndex < ARC_Vector_GetSize(lexer->tokenRules); tokenRuleIndex++){
//get the current token rule
ARC_LexerTokenRule *currentTokenRule = (ARC_LexerTokenRule *)ARC_Vector_Get(lexer->tokenRules, tokenRuleIndex);
//check if the value is smaller than or equal to the minimum value and if it is we can skip it
if(currentTokenRule->id <= minValue){
continue;
}
//check if the value is continous
if(currentTokenRule->id - minValue == 1){
//set the token rule max val to the next most continuous value
lexer->tokenRulesMaxVal = currentTokenRule->id;
//set the next smallest value to check to the the next most continuous value
minValue = currentTokenRule->id;
currentAreContinuous = ARC_True;
break;
}
}
//the current values are not continuous so we can return as token rules are continuous is already set to false
if(currentAreContinuous == ARC_False){
return;
}
//a continuous value was found so loop to next value
}
}
ARC_Bool
#define ARC_Bool
Definition bool.h:10

References ARC_Bool, ARC_DEBUG_LOG_ERROR, arc_errno, ARC_False, ARC_True, ARC_Vector_Add(), ARC_Vector_Get(), ARC_Vector_GetSize(), ARC_LexerTokenRule::id, ARC_Lexer::tokenRules, ARC_Lexer::tokenRulesAreContinuous, and ARC_Lexer::tokenRulesMaxVal.

Referenced by ARC_Lexer_InitBasicTokenRules(), ARC_ParserCSV_InitLexerRulesFn(), ARC_ParserLang_InitLexerRulesFn(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), and TEST_Parser_InitBasicLexerTokenRules().

## ARC_LexerToken_VectorDestroyDataFn()

void ARC_LexerToken_VectorDestroyDataFn | ( | void * | data | ) |

Definition at line 39 of file lexer.c.

{
ARC_LexerToken *token = (ARC_LexerToken *)data;
//deletes the token data string if it exists
if(token->data != NULL){
ARC_String_Destroy(token->data);
}
free(token);
}

References ARC_String_Destroy(), and ARC_LexerToken::data.

Referenced by ARC_Lexer_Create(), and ARC_Lexer_LexString().

## ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule()

ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule | ( | uint32_t | id,
|  | ARC_String * | string )

creates a ARC_LexerTokenRule with a given id and string

Note
  this is intended as a helper funtion #note string will not be freed (it will be copied and the copy will be freed)

Parameters

  [in] | id | a tokens id (basically the token value)
  [in] | character | the string to match against, will be copied

Returns
  a token rule based in the id and string

Definition at line 450 of file lexer.c.

{
//create the token rule
ARC_LexerTokenRule tokenRule;
//set the id
tokenRule.id = id;
//copy and store the automataData (which is just an ARC_String)
ARC_String *automataData;
ARC_String_Copy(&automataData, string);
tokenRule.automataData = (void *)automataData;
//we can use the ARC_Lexer_AutomataMatchCharInStringFn for this
tokenRule.automataFn = ARC_Lexer_AutomataMatchCharInStringFn;
//add the private destroy function
tokenRule.destroyAutomataDataFn = ARC_LexerTokenRule_DestroyStringAutomataDataFn;
//return the created tokenRule
return tokenRule;
}
ARC_LexerTokenRule_DestroyStringAutomataDataFn
void ARC_LexerTokenRule_DestroyStringAutomataDataFn(void *automataData)
Definition lexer.c:424
ARC_Lexer_AutomataMatchCharInStringFn
uint32_t ARC_Lexer_AutomataMatchCharInStringFn(ARC_String **tokenData, ARC_String *string, void *automataData)
checks if the first part of string is a character in substring
Definition lexer.c:356
ARC_String_Copy
void ARC_String_Copy(ARC_String **copy, ARC_String *original)
copy a ARC_String
Definition string.c:62
ARC_LexerTokenRule::destroyAutomataDataFn
ARC_LexerTokenRule_DestroyAutomataDataFn destroyAutomataDataFn
Definition lexer.h:52

References ARC_Lexer_AutomataMatchCharInStringFn(), ARC_LexerTokenRule_DestroyStringAutomataDataFn(), ARC_String_Copy(), ARC_LexerTokenRule::automataData, ARC_LexerTokenRule::automataFn, ARC_LexerTokenRule::destroyAutomataDataFn, and ARC_LexerTokenRule::id.

Referenced by ARC_Lexer_InitBasicTokenRules(), and ARC_ParserLang_InitLexerRulesFn().

## ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween()

ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween | ( | uint32_t | id,
|  | char | start,
|  | char | end )

creates a ARC_LexerTokenRule with a given id and character range

Note
  this is intended as a helper funtion

Parameters

  [in] | id | a tokens id (basically the token value)
  [in] | start | the minimum character value to match against
  [in] | end | the maxamum character value to match against

Returns
  a token rule based in the id and character

Definition at line 400 of file lexer.c.

{
//create the token rule
ARC_LexerTokenRule tokenRule;
//set the id
tokenRule.id = id;
//create and store the automataData (which is just two chars (the minumum and manximum))
char *automataData = (char *)malloc(sizeof(char) * 2);
automataData[0] = start;
automataData[1] = end;
tokenRule.automataData = (void *)automataData;
//we can use the ARC_Lexer_AutomataMatchCharInStringFn for this
tokenRule.automataFn = ARC_Lexer_AutomataMatchCharOrBetweenFn;
//add the private destroy function (we can use the char as it destroys a char pointer of any size)
tokenRule.destroyAutomataDataFn = ARC_LexerTokenRule_DestroyCharAutomataDataFn;
//return the created tokenRule
return tokenRule;
}
ARC_Lexer_AutomataMatchCharOrBetweenFn
uint32_t ARC_Lexer_AutomataMatchCharOrBetweenFn(ARC_String **tokenData, ARC_String *string, void *automataData)
Definition lexer.c:325
ARC_LexerTokenRule_DestroyCharAutomataDataFn
void ARC_LexerTokenRule_DestroyCharAutomataDataFn(void *automataData)
Definition lexer.c:374

References ARC_Lexer_AutomataMatchCharOrBetweenFn(), ARC_LexerTokenRule_DestroyCharAutomataDataFn(), ARC_LexerTokenRule::automataData, ARC_LexerTokenRule::automataFn, ARC_LexerTokenRule::destroyAutomataDataFn, and ARC_LexerTokenRule::id.

Referenced by ARC_Lexer_InitBasicTokenRules(), ARC_ParserCSV_InitLexerRulesFn(), ARC_ParserLang_InitLexerRulesFn(), and TEST_Parser_InitBasicLexerTokenRules().

## ARC_LexerTokenRule_CreateAndReturnMatchCharRule()

ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchCharRule | ( | uint32_t | id,
|  | char | character )

creates a ARC_LexerTokenRule with a given id and character

Note
  this is intended as a helper funtion

Parameters

  [in] | id | a tokens id (basically the token value)
  [in] | character | the character to match against

Returns
  a token rule based in the id and character

Definition at line 378 of file lexer.c.

{
//create the token rule
ARC_LexerTokenRule tokenRule;
//set the id
tokenRule.id = id;
//create and store the automataData (which is just a char)
char *automataData = (char *)malloc(sizeof(char));
*automataData = character;
tokenRule.automataData = (void *)automataData;
//we can use the ARC_Lexer_AutomataMatchCharFn for this
tokenRule.automataFn = ARC_Lexer_AutomataMatchCharFn;
//add the private destroy function
tokenRule.destroyAutomataDataFn = ARC_LexerTokenRule_DestroyCharAutomataDataFn;
//return the created tokenRule
return tokenRule;
}
ARC_Lexer_AutomataMatchCharFn
uint32_t ARC_Lexer_AutomataMatchCharFn(ARC_String **tokenData, ARC_String *string, void *automataData)
checks if the first character of string matches the automataData cast as a char
Definition lexer.c:311

References ARC_Lexer_AutomataMatchCharFn(), ARC_LexerTokenRule_DestroyCharAutomataDataFn(), ARC_LexerTokenRule::automataData, ARC_LexerTokenRule::automataFn, ARC_LexerTokenRule::destroyAutomataDataFn, and ARC_LexerTokenRule::id.

Referenced by ARC_Lexer_InitBasicTokenRules(), ARC_ParserCSV_InitLexerRulesFn(), ARC_ParserLang_InitLexerRulesFn(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), ARC_TEST(), and TEST_Parser_InitBasicLexerTokenRules().

## ARC_LexerTokenRule_CreateAndReturnMatchStringRule()

ARC_LexerTokenRule ARC_LexerTokenRule_CreateAndReturnMatchStringRule | ( | uint32_t | id,
|  | ARC_String * | string )

creates a ARC_LexerTokenRule with a given id and string

Note
  this is intended as a helper funtion #note string will not be freed (it will be copied and the copy will be freed)

Parameters

  [in] | id | a tokens id (basically the token value)
  [in] | character | the string to match against, will be copied

Returns
  a token rule based in the id and string

Definition at line 428 of file lexer.c.

{
//create the token rule
ARC_LexerTokenRule tokenRule;
//set the id
tokenRule.id = id;
//copy and store the automataData (which is just an ARC_String)
ARC_String *automataData;
ARC_String_Copy(&automataData, string);
tokenRule.automataData = (void *)automataData;
//we can use the ARC_Lexer_AutomataMatchStringFn for this
tokenRule.automataFn = ARC_Lexer_AutomataMatchStringFn;
//add the private destroy function
tokenRule.destroyAutomataDataFn = ARC_LexerTokenRule_DestroyStringAutomataDataFn;
//return the created tokenRule
return tokenRule;
}
ARC_Lexer_AutomataMatchStringFn
uint32_t ARC_Lexer_AutomataMatchStringFn(ARC_String **tokenData, ARC_String *string, void *automataData)
checks if the substring automataData as an ARC_String matches the first part of string
Definition lexer.c:341

References ARC_Lexer_AutomataMatchStringFn(), ARC_LexerTokenRule_DestroyStringAutomataDataFn(), ARC_String_Copy(), ARC_LexerTokenRule::automataData, ARC_LexerTokenRule::automataFn, ARC_LexerTokenRule::destroyAutomataDataFn, and ARC_LexerTokenRule::id.

Referenced by ARC_ParserLang_InitLexerRulesFn().

## ARC_LexerTokenRule_DestroyCharAutomataDataFn()

void ARC_LexerTokenRule_DestroyCharAutomataDataFn | ( | void * | automataData | ) |

Definition at line 374 of file lexer.c.

{
free((char *)automataData);
}

Referenced by ARC_LexerTokenRule_CreateAndReturnMatchCharOrBetween(), and ARC_LexerTokenRule_CreateAndReturnMatchCharRule().

## ARC_LexerTokenRule_DestroyStringAutomataDataFn()

void ARC_LexerTokenRule_DestroyStringAutomataDataFn | ( | void * | automataData | ) |

Definition at line 424 of file lexer.c.

{
ARC_String_Destroy((ARC_String *)automataData);
}

References ARC_String_Destroy().

Referenced by ARC_LexerTokenRule_CreateAndReturnMatchCharInStringRule(), and ARC_LexerTokenRule_CreateAndReturnMatchStringRule().

## ARC_LexerTokenRule_VectorCompareDataFn()

ARC_Bool ARC_LexerTokenRule_VectorCompareDataFn | ( | void * | dataA,
|  | void * | dataB )

Definition at line 20 of file lexer.c.

{
ARC_LexerTokenRule *tokenRuleA = (ARC_LexerTokenRule *)dataA;
ARC_LexerTokenRule *tokenRuleB = (ARC_LexerTokenRule *)dataB;
if(tokenRuleA->id == tokenRuleB->id){
return ARC_True;
}
return ARC_False;
}

References ARC_False, ARC_True, and ARC_LexerTokenRule::id.

Referenced by ARC_Lexer_Create().

## ARC_LexerTokenRule_VectorDestroyDataFn()

void ARC_LexerTokenRule_VectorDestroyDataFn | ( | void * | data | ) |

Definition at line 32 of file lexer.c.

{
ARC_LexerTokenRule *tokenRule = (ARC_LexerTokenRule *)data;
tokenRule->destroyAutomataDataFn(tokenRule->automataData);
free(tokenRule);
}

References ARC_LexerTokenRule::automataData, and ARC_LexerTokenRule::destroyAutomataDataFn.

Referenced by ARC_Lexer_Create().