Add initial scanner functions

2 files changed, 277 insertions, 39 deletions

diff --git a/src/badlib.h b/src/badlib.h
index 9bab563..a91fcf8 100644
--- a/src/badlib.h
+++ b/src/badlib.h
@@ -571,6 +571,9 @@ str_next(Str *s) {
 char
 str_peek(Str s) {
     assert(s.mem);
+    if (s.size == 0) {
+        return EOF;
+    }
     return *s.mem;
 }
 
@@ -936,7 +939,6 @@ typedef enum {
     FILE_ERR_OK = 0,
     FILE_ERR_CANT_OPEN,
     FILE_ERR_READ_ERR,
-    FILE_ERR_PATH_TOO_BIG,
     FILE_ERR_EMPTY,
     FILE_ERR_NUM,
 } FileErr;
@@ -945,8 +947,7 @@ Str file_err_str[FILE_ERR_NUM] = {
     cstr(""),
     cstr("couldn't open file"),
     cstr("couldn't read file"),
-    cstr("file path too big"),
-    cstr("file is empty"),
+    cstr("empty file"),
 };
 
 typedef struct FileContents {
@@ -958,19 +959,11 @@ typedef struct FileContents {
 FileContents
 platform_read_file(Str path, Arena *a) {
     // Transform Str to cstr.
-    char path_str[KB(1)];
-    if (path.size >= KB(1) - 1) {
-        return (FileContents){
-            .path = path,
-            .err = FILE_ERR_PATH_TOO_BIG,
-        };
-    }
-    memcpy(path_str, path.mem, path.size);
-    path_str[path.size] = 0;
+    Str path_str = str_concat(path, cstr("\0"), a);
 
     // Read the entire file into memory.
     sz file_size = 0;
-    FILE *fp = fopen(path_str, "rb+");
+    FILE *fp = fopen((char*)path_str.mem, "rb+");
     if (!fp) {
         return (FileContents){
             .path = path,
@@ -989,7 +982,6 @@ platform_read_file(Str path, Arena *a) {
             .err = FILE_ERR_EMPTY,
         };
     }
-
     return (FileContents){
         .path = path,
         .data = (Array){.mem = memory, .size = file_size},
diff --git a/src/main.c b/src/main.c
index 2b51d2c..6be1e88 100644
--- a/src/main.c
+++ b/src/main.c
@@ -21,31 +21,275 @@ init(void) {
     log_init_default();
 }
 
+typedef enum TokenType {
+    TOK_UNKNOWN = 0,
+
+    // Parentheses.
+    TOK_LPAREN,
+    TOK_RPAREN,
+    TOK_LSQUARE,
+    TOK_RSQUARE,
+    TOK_LCURLY,
+    TOK_RCURLY,
+
+    // Basic literals.
+    TOK_NUMBER,
+    TOK_SYMBOL,
+    TOK_STRING,
+    TOK_NIL,
+    TOK_TRUE,
+    TOK_FALSE,
+
+    // Keywords.
+    TOK_LET,
+    TOK_SET,
+    TOK_FUN,
+    TOK_STRUCT,
+    TOK_IF,
+    TOK_MATCH,
+    TOK_CASE,
+    TOK_WHILE,
+    TOK_CONTINUE,
+    TOK_BREAK,
+    TOK_RETURN,
+
+    // Arithmetic ops.
+    TOK_ADD,
+    TOK_SUB,
+    TOK_MUL,
+    TOK_DIV,
+    TOK_MOD,
+
+    // Boolean ops.
+    TOK_NOT,
+    TOK_AND,
+    TOK_OR,
+    TOK_EQ,
+    TOK_LT,
+    TOK_GT,
+    TOK_LE,
+    TOK_GE,
+
+    // Bitwise ops.
+    TOK_BITNOT,
+    TOK_BITAND,
+    TOK_BITOR,
+    TOK_BITLSHIFT,
+    TOK_BITRSHIFT,
+
+    // Special ops.
+    TOK_COLON,
+    TOK_DOT,
+    TOK_AT,
+    TOK_ASSIGN,
+
+    // End of file.
+    TOK_EOF,
+} TokenType;
+
+Str token_str[] = {
+    [TOK_UNKNOWN] = cstr("UNKNOWN"),
+
+    // Parentheses.
+    [TOK_LPAREN] = cstr("LPAREN"),
+    [TOK_RPAREN] = cstr("RPAREN"),
+    [TOK_LSQUARE] = cstr("LSQUARE"),
+    [TOK_RSQUARE] = cstr("RSQUARE"),
+    [TOK_LCURLY] = cstr("LCURLY"),
+    [TOK_RCURLY] = cstr("RCURLY"),
+
+    // Basic literals.
+    [TOK_NUMBER] = cstr("NUMBER"),
+    [TOK_SYMBOL] = cstr("SYMBOL"),
+    [TOK_STRING] = cstr("STRING"),
+    [TOK_NIL] = cstr("NIL"),
+    [TOK_TRUE] = cstr("TRUE"),
+    [TOK_FALSE] = cstr("FALSE"),
+
+    // Keywords.
+    [TOK_LET] = cstr("LET"),
+    [TOK_SET] = cstr("SET"),
+    [TOK_FUN] = cstr("FUN"),
+    [TOK_STRUCT] = cstr("STRUCT"),
+    [TOK_IF] = cstr("IF"),
+    [TOK_MATCH] = cstr("MATCH"),
+    [TOK_CASE] = cstr("CASE"),
+    [TOK_WHILE] = cstr("WHILE"),
+    [TOK_CONTINUE] = cstr("CONTINUE"),
+    [TOK_BREAK] = cstr("BREAK"),
+    [TOK_RETURN] = cstr("RETURN"),
+
+    // Arithmetic ops.
+    [TOK_ADD] = cstr("ADD"),
+    [TOK_SUB] = cstr("SUB"),
+    [TOK_MUL] = cstr("MUL"),
+    [TOK_DIV] = cstr("DIV"),
+    [TOK_MOD] = cstr("MOD"),
+
+    // Boolean ops.
+    [TOK_NOT] = cstr("NOT"),
+    [TOK_AND] = cstr("AND"),
+    [TOK_OR] = cstr("OR"),
+    [TOK_EQ] = cstr("EQ"),
+    [TOK_LT] = cstr("LT"),
+    [TOK_GT] = cstr("GT"),
+    [TOK_LE] = cstr("LE"),
+    [TOK_GE] = cstr("GE"),
+
+    // Bitwise ops.
+    [TOK_BITNOT] = cstr("BITNOT"),
+    [TOK_BITAND] = cstr("BITAND"),
+    [TOK_BITOR] = cstr("BITOR"),
+    [TOK_BITLSHIFT] = cstr("BITLSHIFT"),
+    [TOK_BITRSHIFT] = cstr("BITRSHIFT"),
+
+    // Special ops.
+    [TOK_COLON] = cstr("COLON"),
+    [TOK_DOT] = cstr("DOT"),
+    [TOK_AT] = cstr("AT"),
+    [TOK_ASSIGN] = cstr("ASSIGN"),
+
+    // End of file.
+    [TOK_EOF] = cstr("EOF"),
+};
+
+typedef struct Token {
+    TokenType type;
+    Str val;
+    sz line;
+    sz col;
+} Token;
+
+typedef struct Scanner {
+    Str str;
+    sz line;
+    sz col;
+    Arena *storage;
+} Scanner;
+
+char
+scan_next(Scanner *scanner) {
+    char c = str_next(&scanner->str);
+    if (c == '\n') {
+        scanner->line++;
+        scanner->col = 0;
+    } else {
+        scanner->col++;
+    }
+    return c;
+}
+
+bool
+scan_has_next(Scanner *scanner) {
+    return scanner->str.size;
+}
+
+char
+scan_peek(Scanner *scanner) {
+    return str_peek(scanner->str);
+}
+
+Token
+emit_token(Scanner *scanner, TokenType t) {
+    return (Token){
+        .line = scanner->line + 1,
+        .col = scanner->col + 1,
+        .type = t,
+    };
+}
+
+Token
+emit_token_err(Scanner *scanner, Str err_msg) {
+    return (Token){
+        .line = scanner->line + 1,
+        .col = scanner->col + 1,
+        .val = err_msg,
+        .type = TOK_UNKNOWN,
+    };
+}
+
+void
+scan_skip_line(Scanner *scanner) {
+    SearchResult newline = array_find_next(scanner->str, cstr("\n"));
+    if (newline.found) {
+        scanner->str.mem += newline.pos + 1;
+        scanner->str.size -= newline.pos + 1;
+        scanner->line++;
+        scanner->col = 0;
+    }
+}
+
+void
+scan_skip_whitespace(Scanner *scanner) {
+    while (scan_has_next(scanner)) {
+        char c = scan_peek(scanner);
+        switch (c) {
+            case ' ':
+            case ',': // We are currently considering commas as syntactic sugar.
+            case '\f':
+            case '\n':
+            case '\r':
+            case '\t':
+            case '\v': {
+                scan_next(scanner);
+            } break;
+            // Found a comment! (skip)
+            case ';': {
+                scan_skip_line(scanner);
+            } break;
+            default: {
+                return;
+            } break;
+        }
+    }
+}
+
+Token
+scan_token(Scanner *scanner) {
+    assert(scanner);
+
+    scan_skip_whitespace(scanner);
+    if (!scan_has_next(scanner)) {
+        return emit_token(scanner, TOK_EOF);
+    }
+
+    Scanner current = *scanner;
+    char c = scan_next(scanner);
+    (void)c;
+    // TODO: rest of the operations...
+
+    // At this point we have an error, find the next newline.
+    scan_skip_line(scanner);
+    return emit_token_err(&current, cstr("unexpected character"));
+}
+
 void
 process_file(Str path) {
-    (void)path;
-    // println("processing: ");
-    // println("%s", path);
-    // println("initialing memory...");
     Arena lexer_arena = arena_create(LEXER_MEM, os_allocator);
-    u8 *mem;
-    mem = arena_malloc(16, &lexer_arena);
-    println("%{Arena}", &lexer_arena);
-    mem[0] = 255;
-    mem[1] = 255;
-    mem[2] = 0xba;
-    mem[3] = 0xdd;
-    mem[4] = 0x10;
-    mem[5] = 0xde;
-    mem[6] = 255;
-    mem[7] = 255;
-    println("%{Arena}", &lexer_arena);
-    arena_malloc(8, &lexer_arena);
-    mem = arena_realloc(mem, 16, 32, &lexer_arena);
-    // arena_free(mem, 32, &lexer_arena);
-    println("%{Arena}", &lexer_arena);
-    // println("reading file...");
-    // FileContents file = platform_read_file(path, &lexer_arena);
+
+    FileContents file = platform_read_file(path, &lexer_arena);
+    if (file.err) {
+        printf("file.err: %d\n", file.err);
+        eprintln("%s: error: %s", path, cstr("WOT"));
+        return;
+    }
+
+    Scanner scanner = {
+        .str = file.data,
+        .storage = &lexer_arena,
+    };
+    Token tok = {0};
+    while (tok.type != TOK_EOF) {
+        tok = scan_token(&scanner);
+        println("%s:%d:%d:%s %s", path, tok.line, tok.col, token_str[tok.type],
+                tok.val);
+    }
+    // while (true) {
+    //     Token tok = scan_token(&scanner);
+    //     println("%s:%d:%d:%s %s", path, tok.line, tok.col, token_str[tok.type],
+    //             tok.val);
+    //     if (tok.type == TOK_EOF) break;
+    // }
 
     // Str scanner = file.data;
     // // NOTE: Testing file read line by line.
@@ -57,10 +301,12 @@ process_file(Str path) {
     // println("<<< %x{4} %b{4} %f{2} %s %{Arena} >>>", 123, 3, 1.345,
     //         cstr("BOOM!"), &logger_inf.storage);
 
-    println("%{Mem}", &(Array){lexer_arena.beg, lexer_arena.size});
-    eprintln("%s:%d:%d: %s -> %c", path, 1, 1, cstr("error: testing string logger"), 'X');
-    // while (true) {}
+    // println("%{Mem}", &(Array){lexer_arena.beg, lexer_arena.size});
+    // eprintln("%s:%d:%d: %s -> %c", path, 1, 1, cstr("error: testing string
+    // logger"), 'X'); while (true) {}
     // TODO: run lexer.
+    // Free up resources.
+    arena_destroy(&lexer_arena, os_allocator);
 }
 
 #ifndef BIN_NAME