1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
#ifndef BDL_LEXER_H
#define BDL_LEXER_H
#include "string_view.h"
typedef enum TokenType {
TOKEN_UNKNOWN = 0,
// Parentheses.
TOKEN_LPAREN,
TOKEN_RPAREN,
// Primitive types.
TOKEN_FIXNUM,
TOKEN_SYMBOL,
TOKEN_STRING,
TOKEN_NIL,
TOKEN_TRUE,
TOKEN_FALSE,
// Keywords.
TOKEN_LAMBDA,
TOKEN_IF,
TOKEN_DEF,
TOKEN_SET,
TOKEN_FUN,
// End of file.
TOKEN_EOF,
} TokenType;
typedef struct Token {
TokenType type;
StringView value;
size_t line;
size_t col;
} Token;
typedef struct Scanner {
StringView current;
size_t line_number;
size_t col_number;
size_t offset;
} Scanner;
// Print a token to standard output for debugging purposes.
void print_token(Token tok);
// Same functionality as the ScanView pairs, but keeping track of line and
// column numbers.
char scan_next(Scanner *scanner);
char scan_peek(const Scanner *scanner);
// Check if the current scanner still have characters left.
bool scan_has_next(const Scanner *scanner);
// Advance the scanner until we ran out of whitespace.
void skip_whitespace(Scanner *scanner);
// Check if a given character is a delimiter.
bool is_delimiter(char c);
// Extract the token type from the current string.
TokenType find_primitive_type(const StringView value);
// Generate a list of tokens from the given string.
Token * tokenize(const StringView *sv, Errors *errors);
#endif // BDL_LEXER_H
|
