1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
#ifndef BDL_LEXER_H
#define BDL_LEXER_H
typedef enum TokenType {
TOKEN_UNKNOWN = 0,
TOKEN_LPAREN,
TOKEN_RPAREN,
TOKEN_QUOTE,
TOKEN_TRUE,
TOKEN_FALSE,
TOKEN_NIL,
TOKEN_FIXNUM,
TOKEN_SYMBOL,
TOKEN_STRING,
TOKEN_EOF,
} TokenType;
typedef struct Token {
TokenType type;
StringView value;
size_t line;
size_t column;
} Token;
typedef struct Tokens {
Token *buf;
size_t size;
size_t cap;
} Tokens;
typedef struct Scanner {
StringView current;
size_t line_number;
size_t col_number;
size_t offset;
} Scanner;
// Print a token to standard output for debugging purposes.
void print_token(Token tok);
// Push a token to the token list.
void push_token(Tokens *tokens, Token tok);
// Same functionality as the ScanView pairs, but keeping track of line and
// column numbers.
char scan_next(Scanner *scanner);
char scan_peek(const Scanner *scanner);
// Check if the current scanner still have characters left.
bool scan_has_next(const Scanner *scanner);
// Advance the scanner until we ran out of whitespace.
void skip_whitespace(Scanner *scanner);
// Check if a given character is a delimiter.
bool is_delimiter(char c);
// Extract the token type from the current string.
TokenType find_primitive_type(const StringView value);
// Generate a list of tokens from the given string.
Tokens tokenize(const StringView *sv);
#define TOK_BUF_CAP 256
#endif // BDL_LEXER_H
|
