Prepare third compiler implementation

1 files changed, 67 insertions, 0 deletions

diff --git a/src/lexer.h b/src/lexer.h
new file mode 100755
index 0000000..e56f5f2
--- /dev/null
+++ b/src/lexer.h
@@ -0,0 +1,67 @@
+#ifndef BDL_LEXER_H
+#define BDL_LEXER_H
+
+#include "string_view.h"
+
+typedef enum TokenType {
+    TOKEN_UNKNOWN = 0,
+
+    // Parentheses.
+    TOKEN_LPAREN,
+    TOKEN_RPAREN,
+
+    // Primitive types.
+    TOKEN_FIXNUM,
+    TOKEN_SYMBOL,
+    TOKEN_STRING,
+    TOKEN_NIL,
+    TOKEN_TRUE,
+    TOKEN_FALSE,
+
+    // End of file.
+    TOKEN_EOF,
+} TokenType;
+
+typedef struct Token {
+    TokenType type;
+    StringView value;
+    size_t line;
+    size_t column;
+} Token;
+
+typedef struct Tokens {
+    Token *tokens;
+    Errors errors;
+} Tokens;
+
+typedef struct Scanner {
+    StringView current;
+    size_t line_number;
+    size_t col_number;
+    size_t offset;
+} Scanner;
+
+// Print a token to standard output for debugging purposes.
+void print_token(Token tok);
+
+// Same functionality as the ScanView pairs, but keeping track of line and
+// column numbers.
+char scan_next(Scanner *scanner);
+char scan_peek(const Scanner *scanner);
+
+// Check if the current scanner still have characters left.
+bool scan_has_next(const Scanner *scanner);
+
+// Advance the scanner until we ran out of whitespace.
+void skip_whitespace(Scanner *scanner);
+
+// Check if a given character is a delimiter.
+bool is_delimiter(char c);
+
+// Extract the token type from the current string.
+TokenType find_primitive_type(const StringView value);
+
+// Generate a list of tokens from the given string.
+Tokens tokenize(const StringView *sv);
+
+#endif // BDL_LEXER_H