Ensure EOF is handled properly

2 files changed, 42 insertions, 62 deletions

diff --git a/src/lexer.c b/src/lexer.c
index 56b670b..80ef3ad 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -262,6 +262,10 @@ tokenize(const StringView *sv) {
             case '.': { token.type = TOKEN_DOT; } break;
             case '@': { token.type = TOKEN_AT; } break;
             default: {
+                if (c == EOF || c == '\0') {
+                    token.type = TOKEN_EOF;
+                    break;
+                }
                 size_t n = 1;
                 if (c == '-' && !is_delimiter(scan_peek(&scanner))) {
                     n += scan_number_token(&scanner);
@@ -304,14 +308,5 @@ tokenize(const StringView *sv) {
         }
         array_push(tokens, token);
     }
-
-    // Push EOF token.
-    Token token =  (Token){
-        .type = TOKEN_EOF,
-        .line = scanner.line_number,
-        .col = 1,
-    };
-    array_push(tokens, token);
-
     return tokens;
 }
diff --git a/tests/constants/numbers.bdl b/tests/constants/numbers.bdl
index 0412af8..136b5fb 100644
--- a/tests/constants/numbers.bdl
+++ b/tests/constants/numbers.bdl
@@ -1,56 +1,41 @@
 ; Signed integers.
-3
-; -1
-0.1
-43
-0
--123
-0x123
-0xabCd
-0xABCDEF
--0x12
-; a
-; fun
-; a.a
-; a.1  ; should err
-; 1.a  ; should err
-; (print 1:s8)
-; (print 52:s8)
-; (print -1:s8)
-; (print 0:s8)
-; (print 1:s16)
-; (print 52:s16)
-; (print -1:s16)
-; (print 0:s16)
-; (print 1:s32)
-; (print 52:s32)
-; (print -1:s32)
-; (print 0:s32)
-; (print 1:s64)
-; (print 52:s64)
-; (print -1:s64)
-; (print 0:s64)
+(print 1:s8)
+(print 52:s8)
+(print -1:s8)
+(print 0:s8)
+(print 1:s16)
+(print 52:s16)
+(print -1:s16)
+(print 0:s16)
+(print 1:s32)
+(print 52:s32)
+(print -1:s32)
+(print 0:s32)
+(print 1:s64)
+(print 52:s64)
+(print -1:s64)
+(print 0:s64)
 
-; ; Unsigned integers.
-; (print 1:u8)
-; (print 52:u8)
-; (print -1:u8)
-; (print 0:u8)
-; (print 1:u16)
-; (print 52:u16)
-; (print -1:u16)
-; (print 0:u16)
-; (print 1:u32)
-; (print 52:u32)
-; (print -1:u32)
-; (print 0:u32)
-; (print 1:u64)
-; (print 52:u64)
-; (print -1:u64)
-; (print 0:u64)
+; Unsigned integers.
+(print 1:u8)
+(print 52:u8)
+(print -1:u8)
+(print 0:u8)
+(print 1:u16)
+(print 52:u16)
+(print -1:u16)
+(print 0:u16)
+(print 1:u32)
+(print 52:u32)
+(print -1:u32)
+(print 0:u32)
+(print 1:u64)
+(print 52:u64)
+(print -1:u64)
+(print 0:u64)
 
-; ; Type inference.
-; (print 1)
-; (print 52)
-; (print -1)
-; (print 0)
+; Type inference.
+(print 1)
+(print 52)
+(print -1)
+(print 0)