1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
#ifndef BDL_LEXER_H
#define BDL_LEXER_H
#include "string_view.h"
typedef enum TokenType {
TOKEN_UNKNOWN = 0,
// Parentheses.
TOKEN_LPAREN,
TOKEN_RPAREN,
TOKEN_LSQUARE,
TOKEN_RSQUARE,
TOKEN_LCURLY,
TOKEN_RCURLY,
// Primitive types.
TOKEN_NUMBER,
TOKEN_SYMBOL,
TOKEN_STRING,
TOKEN_NIL,
TOKEN_TRUE,
TOKEN_FALSE,
// Keywords.
TOKEN_LAMBDA,
TOKEN_IF,
TOKEN_DEF,
TOKEN_SET,
TOKEN_FUN,
TOKEN_STRUCT,
// Arithmetic ops.
TOKEN_ADD,
TOKEN_SUB,
TOKEN_MUL,
TOKEN_DIV,
TOKEN_MOD,
// Boolean operations.
TOKEN_NOT,
TOKEN_AND,
TOKEN_OR,
TOKEN_EQ,
TOKEN_LT,
TOKEN_GT,
TOKEN_LE,
TOKEN_GE,
// Special operators.
TOKEN_COLON,
TOKEN_DOT,
TOKEN_AT,
// End of file.
TOKEN_EOF,
} TokenType;
typedef struct Token {
TokenType type;
StringView value;
size_t line;
size_t col;
} Token;
typedef struct Scanner {
StringView current;
size_t line_number;
size_t col_number;
size_t offset;
} Scanner;
// Print a token to standard output for debugging purposes.
void print_token(Token tok);
// Same functionality as with StringView, but keeping track of line and column
// numbers.
char scan_next(Scanner *scanner);
char scan_peek(const Scanner *scanner);
// Check if the current scanner still have characters left.
bool scan_has_next(const Scanner *scanner);
// Advance the scanner until we ran out of whitespace.
void skip_whitespace(Scanner *scanner);
// Check if a given character is a delimiter.
bool is_delimiter(char c);
// Extract the token type from the current string.
TokenType find_token_type(const StringView value);
// Generate a list of tokens from the given string.
Token * tokenize(const StringView *sv);
// Display tokens from token list.
void print_tokens(Token *tokens);
#endif // BDL_LEXER_H
|
