#ifndef BDL_LEXER_H #define BDL_LEXER_H typedef enum TokenType { TOKEN_UNKNOWN = 0, TOKEN_LPAREN, TOKEN_RPAREN, TOKEN_QUOTE, TOKEN_TRUE, TOKEN_FALSE, TOKEN_NIL, TOKEN_FIXNUM, TOKEN_SYMBOL, TOKEN_STRING, TOKEN_EOF, } TokenType; typedef struct Token { TokenType type; StringView value; size_t line; size_t column; } Token; typedef struct Tokens { Token *buf; size_t size; size_t cap; } Tokens; typedef struct Scanner { StringView current; size_t line_number; size_t col_number; size_t offset; } Scanner; // Print a token to standard output for debugging purposes. void print_token(Token tok); // Push a token to the token list. void push_token(Tokens *tokens, Token tok); // Same functionality as the ScanView pairs, but keeping track of line and // column numbers. char scan_next(Scanner *scanner); char scan_peek(const Scanner *scanner); // Check if the current scanner still have characters left. bool scan_has_next(const Scanner *scanner); // Advance the scanner until we ran out of whitespace. void skip_whitespace(Scanner *scanner); // Check if a given character is a delimiter. bool is_delimiter(char c); // Extract the token type from the current string. TokenType find_primitive_type(const StringView value); // Generate a list of tokens from the given string. Tokens tokenize(const StringView *sv); #define TOK_BUF_CAP 256 #endif // BDL_LEXER_H