Add new syntax to lexer and prepare refactor

1 files changed, 18 insertions, 5 deletions

diff --git a/src/lexer.h b/src/lexer.h
index c477fbd..d864a1d 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -9,9 +9,13 @@ typedef enum TokenType {
     // Parentheses.
     TOKEN_LPAREN,
     TOKEN_RPAREN,
+    TOKEN_LSQUARE,
+    TOKEN_RSQUARE,
+    TOKEN_LCURLY,
+    TOKEN_RCURLY,
 
     // Primitive types.
-    TOKEN_FIXNUM,
+    TOKEN_NUMBER,
     TOKEN_SYMBOL,
     TOKEN_STRING,
     TOKEN_NIL,
@@ -24,6 +28,12 @@ typedef enum TokenType {
     TOKEN_DEF,
     TOKEN_SET,
     TOKEN_FUN,
+    TOKEN_STRUCT,
+
+    // Special operators.
+    TOKEN_COLON,
+    TOKEN_DOT,
+    TOKEN_AT,
 
     // End of file.
     TOKEN_EOF,
@@ -46,8 +56,8 @@ typedef struct Scanner {
 // Print a token to standard output for debugging purposes.
 void print_token(Token tok);
 
-// Same functionality as the ScanView pairs, but keeping track of line and
-// column numbers.
+// Same functionality as with StringView, but keeping track of line and column
+// numbers.
 char scan_next(Scanner *scanner);
 char scan_peek(const Scanner *scanner);
 
@@ -61,9 +71,12 @@ void skip_whitespace(Scanner *scanner);
 bool is_delimiter(char c);
 
 // Extract the token type from the current string.
-TokenType find_primitive_type(const StringView value);
+TokenType find_token_type(const StringView value);
 
 // Generate a list of tokens from the given string.
-Token * tokenize(const StringView *sv, Errors *errors);
+Token * tokenize(const StringView *sv);
+
+// Display tokens from token list.
+void print_tokens(Token *tokens);
 
 #endif // BDL_LEXER_H