#ifndef BDL_LEXER_H #define BDL_LEXER_H #include "string_view.h" typedef enum TokenType { TOKEN_UNKNOWN = 0, // Parentheses. TOKEN_LPAREN, TOKEN_RPAREN, TOKEN_LSQUARE, TOKEN_RSQUARE, TOKEN_LCURLY, TOKEN_RCURLY, // Primitive types. TOKEN_NUMBER, TOKEN_SYMBOL, TOKEN_STRING, TOKEN_NIL, TOKEN_TRUE, TOKEN_FALSE, // Keywords. TOKEN_LAMBDA, TOKEN_IF, TOKEN_DEF, TOKEN_SET, TOKEN_FUN, TOKEN_STRUCT, // Arithmetic ops. TOKEN_ADD, TOKEN_SUB, TOKEN_MUL, TOKEN_DIV, TOKEN_MOD, // Boolean operations. TOKEN_NOT, TOKEN_AND, TOKEN_OR, TOKEN_EQ, TOKEN_LT, TOKEN_GT, TOKEN_LE, TOKEN_GE, // Special operators. TOKEN_COLON, TOKEN_DOT, TOKEN_AT, // End of file. TOKEN_EOF, } TokenType; typedef struct Token { TokenType type; StringView value; size_t line; size_t col; } Token; typedef struct Scanner { StringView current; size_t line_number; size_t col_number; size_t offset; } Scanner; // Print a token to standard output for debugging purposes. void print_token(Token tok); // Same functionality as with StringView, but keeping track of line and column // numbers. char scan_next(Scanner *scanner); char scan_peek(const Scanner *scanner); // Check if the current scanner still have characters left. bool scan_has_next(const Scanner *scanner); // Advance the scanner until we ran out of whitespace. void skip_whitespace(Scanner *scanner); // Check if a given character is a delimiter. bool is_delimiter(char c); // Extract the token type from the current string. TokenType find_token_type(const StringView value); // Generate a list of tokens from the given string. Token * tokenize(const StringView *sv); // Display tokens from token list. void print_tokens(Token *tokens); #endif // BDL_LEXER_H