1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
|
#ifndef BDL_LEXER_H
#define BDL_LEXER_H
#include "string_view.h"
typedef enum TokenType {
TOKEN_UNKNOWN = 0,
// Parentheses.
TOKEN_LPAREN,
TOKEN_RPAREN,
// Primitive types.
TOKEN_FIXNUM,
TOKEN_SYMBOL,
TOKEN_STRING,
// Keywords.
TOKEN_NIL,
TOKEN_QUOTE,
TOKEN_TRUE,
TOKEN_FALSE,
TOKEN_IF,
TOKEN_ELSE,
TOKEN_DEF,
TOKEN_FUN,
TOKEN_LAMBDA,
// Arithmetic.
TOKEN_ADD,
TOKEN_MIN,
TOKEN_MUL,
TOKEN_DIV,
TOKEN_MOD,
// Boolean comparisons.
TOKEN_NOT,
TOKEN_AND,
TOKEN_OR,
TOKEN_EQUAL,
TOKEN_LESS_THAN,
TOKEN_GREATER_THAN,
TOKEN_LESS_EQUAL_THAN,
TOKEN_GREATER_EQUAL_THAN,
TOKEN_EOF,
} TokenType;
typedef struct Token {
TokenType type;
StringView value;
size_t line;
size_t column;
} Token;
typedef struct Scanner {
StringView current;
size_t line_number;
size_t col_number;
size_t offset;
} Scanner;
// Print a token to standard output for debugging purposes.
void print_token(Token tok);
// Same functionality as the ScanView pairs, but keeping track of line and
// column numbers.
char scan_next(Scanner *scanner);
char scan_peek(const Scanner *scanner);
// Check if the current scanner still have characters left.
bool scan_has_next(const Scanner *scanner);
// Advance the scanner until we ran out of whitespace.
void skip_whitespace(Scanner *scanner);
// Check if a given character is a delimiter.
bool is_delimiter(char c);
// Extract the token type from the current string.
TokenType find_primitive_type(const StringView value);
// Generate a list of tokens from the given string.
Token * tokenize(const StringView *sv);
#define TOK_BUF_CAP 256
#endif // BDL_LEXER_H
|
