Add many scanning functions

1 files changed, 301 insertions, 43 deletions

diff --git a/src/main.c b/src/main.c
index 6be1e88..61d02a7 100644
--- a/src/main.c
+++ b/src/main.c
@@ -36,9 +36,6 @@ typedef enum TokenType {
     TOK_NUMBER,
     TOK_SYMBOL,
     TOK_STRING,
-    TOK_NIL,
-    TOK_TRUE,
-    TOK_FALSE,
 
     // Keywords.
     TOK_LET,
@@ -52,36 +49,40 @@ typedef enum TokenType {
     TOK_CONTINUE,
     TOK_BREAK,
     TOK_RETURN,
+    TOK_NIL,
+    TOK_TRUE,
+    TOK_FALSE,
 
     // Arithmetic ops.
-    TOK_ADD,
-    TOK_SUB,
-    TOK_MUL,
-    TOK_DIV,
-    TOK_MOD,
-
-    // Boolean ops.
-    TOK_NOT,
-    TOK_AND,
-    TOK_OR,
-    TOK_EQ,
-    TOK_LT,
-    TOK_GT,
-    TOK_LE,
-    TOK_GE,
+    TOK_ADD,  // +
+    TOK_SUB,  // -
+    TOK_MUL,  // *
+    TOK_DIV,  // /
+    TOK_MOD,  // %
+
+    // Logical ops.
+    TOK_NOT,    // !
+    TOK_AND,    // &&
+    TOK_OR,     // ||
+    TOK_EQ,     // ==
+    TOK_NOTEQ,  // !=
+    TOK_LT,     // <
+    TOK_GT,     // >
+    TOK_LE,     // <=
+    TOK_GE,     // >=
 
     // Bitwise ops.
-    TOK_BITNOT,
-    TOK_BITAND,
-    TOK_BITOR,
-    TOK_BITLSHIFT,
-    TOK_BITRSHIFT,
+    TOK_BITNOT,     // ~
+    TOK_BITAND,     // &
+    TOK_BITOR,      // |
+    TOK_BITLSHIFT,  // <<
+    TOK_BITRSHIFT,  // >>
 
     // Special ops.
-    TOK_COLON,
-    TOK_DOT,
-    TOK_AT,
-    TOK_ASSIGN,
+    TOK_COLON,   // :
+    TOK_DOT,     // .
+    TOK_AT,      // @
+    TOK_ASSIGN,  // =
 
     // End of file.
     TOK_EOF,
@@ -102,9 +103,6 @@ Str token_str[] = {
     [TOK_NUMBER] = cstr("NUMBER"),
     [TOK_SYMBOL] = cstr("SYMBOL"),
     [TOK_STRING] = cstr("STRING"),
-    [TOK_NIL] = cstr("NIL"),
-    [TOK_TRUE] = cstr("TRUE"),
-    [TOK_FALSE] = cstr("FALSE"),
 
     // Keywords.
     [TOK_LET] = cstr("LET"),
@@ -118,6 +116,9 @@ Str token_str[] = {
     [TOK_CONTINUE] = cstr("CONTINUE"),
     [TOK_BREAK] = cstr("BREAK"),
     [TOK_RETURN] = cstr("RETURN"),
+    [TOK_NIL] = cstr("NIL"),
+    [TOK_TRUE] = cstr("TRUE"),
+    [TOK_FALSE] = cstr("FALSE"),
 
     // Arithmetic ops.
     [TOK_ADD] = cstr("ADD"),
@@ -126,11 +127,12 @@ Str token_str[] = {
     [TOK_DIV] = cstr("DIV"),
     [TOK_MOD] = cstr("MOD"),
 
-    // Boolean ops.
+    // Logical ops.
     [TOK_NOT] = cstr("NOT"),
     [TOK_AND] = cstr("AND"),
     [TOK_OR] = cstr("OR"),
     [TOK_EQ] = cstr("EQ"),
+    [TOK_NOTEQ] = cstr("NOTEQ"),
     [TOK_LT] = cstr("LT"),
     [TOK_GT] = cstr("GT"),
     [TOK_LE] = cstr("LE"),
@@ -190,10 +192,14 @@ scan_peek(Scanner *scanner) {
 }
 
 Token
-emit_token(Scanner *scanner, TokenType t) {
+emit_token(Scanner current, Scanner *scanner, TokenType t) {
+    Str val = current.str;
+    val.size = current.str.size - scanner->str.size;
+    val.size = val.size < 0 ? 0 : val.size;
     return (Token){
-        .line = scanner->line + 1,
-        .col = scanner->col + 1,
+        .val = val,
+        .line = current.line + 1,
+        .col = current.col + 1,
         .type = t,
     };
 }
@@ -225,7 +231,7 @@ scan_skip_whitespace(Scanner *scanner) {
         char c = scan_peek(scanner);
         switch (c) {
             case ' ':
-            case ',': // We are currently considering commas as syntactic sugar.
+            case ',':  // Commas are just syntactic sugar.
             case '\f':
             case '\n':
             case '\r':
@@ -244,22 +250,273 @@ scan_skip_whitespace(Scanner *scanner) {
     }
 }
 
+bool
+is_valid_split(char c) {
+    switch (c) {
+        case ';':
+        case '(':
+        case ')':
+        case '[':
+        case ']':
+        case '{':
+        case '}':
+        case '+':
+        case '-':
+        case '*':
+        case '/':
+        case '%':
+        case '!':
+        case '=':
+        case '<':
+        case '>':
+        case '~':
+        case '&':
+        case '|':
+        case ':':
+        case '.':
+        case '@':
+        case '"':
+        case ' ':
+        case ',':
+        case '\f':
+        case '\n':
+        case '\r':
+        case '\t':
+        case '\v': {
+            return true;
+        } break;
+    }
+    return false;
+}
+
+void
+scan_skip_until_valid(Scanner *scanner) {
+    while (scan_has_next(scanner)) {
+        char c = scan_peek(scanner);
+        if (is_valid_split(c)) {
+            return;
+        }
+        scan_next(scanner);
+    }
+}
+
+Token
+emit_token_number(Scanner *scanner) {
+    Scanner current = *scanner;
+    char c = scan_peek(scanner);
+    if (c == '+' || c == '-') {
+        scan_next(scanner);
+        if (str_has_prefix(scanner->str, cstr("0b")) ||
+            str_has_prefix(scanner->str, cstr("0x"))) {
+            scan_skip_until_valid(scanner);
+            return emit_token_err(
+                &current,
+                cstr("malformed number: binary/hex numbers can't be signed"));
+        }
+    }
+    if (str_has_prefix(scanner->str, cstr("0b"))) {
+        scan_next(scanner);
+        scan_next(scanner);
+        while (scan_has_next(scanner)) {
+            char c = scan_peek(scanner);
+            if (c == '0' || c == '1') {
+                scan_next(scanner);
+                continue;
+            }
+            if (is_valid_split(c)) {
+                break;
+            }
+            scan_skip_until_valid(scanner);
+            return emit_token_err(
+                &current, cstr("malformed number: invalid binary number"));
+        }
+    } else if (str_has_prefix(scanner->str, cstr("0x"))) {
+        scan_next(scanner);
+        scan_next(scanner);
+        while (scan_has_next(scanner)) {
+            char c = scan_peek(scanner);
+            if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
+                (c >= 'A' && c <= 'F')) {
+                scan_next(scanner);
+                continue;
+            }
+            if (is_valid_split(c)) {
+                break;
+            }
+            scan_skip_until_valid(scanner);
+            return emit_token_err(&current,
+                                  cstr("malformed number: invalid hex number"));
+        }
+    } else {
+        // TODO: handle fractions, signs and exponents properly.
+        while (scan_has_next(scanner)) {
+            char c = scan_peek(scanner);
+            switch (c) {
+                case '+':
+                case '-':
+                case 'e':
+                case 'E':
+                case '.':
+                    scan_next(scanner);
+                    continue;
+            }
+            if (c >= '0' && c <= '9') {
+                scan_next(scanner);
+                continue;
+            }
+            break;
+        }
+    }
+    return emit_token(current, scanner, TOK_NUMBER);
+}
+
 Token
 scan_token(Scanner *scanner) {
     assert(scanner);
 
     scan_skip_whitespace(scanner);
     if (!scan_has_next(scanner)) {
-        return emit_token(scanner, TOK_EOF);
+        return emit_token(*scanner, scanner, TOK_EOF);
     }
 
     Scanner current = *scanner;
     char c = scan_next(scanner);
-    (void)c;
-    // TODO: rest of the operations...
+    switch (c) {
+        case '(':
+            return emit_token(current, scanner, TOK_LPAREN);
+        case ')':
+            return emit_token(current, scanner, TOK_RPAREN);
+        case '[':
+            return emit_token(current, scanner, TOK_LSQUARE);
+        case ']':
+            return emit_token(current, scanner, TOK_RSQUARE);
+        case '{':
+            return emit_token(current, scanner, TOK_LCURLY);
+        case '}':
+            return emit_token(current, scanner, TOK_RCURLY);
+        case '+': {
+            char p = scan_peek(scanner);
+            if (p >= '0' && p <= '9') {
+                *scanner = current;
+                return emit_token_number(scanner);
+            }
+            return emit_token(current, scanner, TOK_ADD);
+        };
+        case '-': {
+            char p = scan_peek(scanner);
+            if (p >= '0' && p <= '9') {
+                *scanner = current;
+                return emit_token_number(scanner);
+            }
+            return emit_token(current, scanner, TOK_ADD);
+        };
+        case '*':
+            return emit_token(current, scanner, TOK_MUL);
+        case '/':
+            return emit_token(current, scanner, TOK_DIV);
+        case '%':
+            return emit_token(current, scanner, TOK_MOD);
+        case '!': {
+            if (scan_peek(scanner) == '=') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_NOTEQ);
+            }
+            return emit_token(current, scanner, TOK_NOT);
+        };
+        case '=': {
+            if (scan_peek(scanner) == '=') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_EQ);
+            }
+            return emit_token(current, scanner, TOK_ASSIGN);
+        };
+        case '<': {
+            char p = scan_peek(scanner);
+            if (p == '=') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_LE);
+            }
+            if (p == '<') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_BITLSHIFT);
+            }
+            return emit_token(current, scanner, TOK_LT);
+        };
+        case '>': {
+            char p = scan_peek(scanner);
+            if (p == '=') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_GE);
+            }
+            if (p == '>') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_BITRSHIFT);
+            }
+            return emit_token(current, scanner, TOK_GT);
+        };
+        case '~':
+            return emit_token(current, scanner, TOK_BITNOT);
+        case '&': {
+            if (scan_peek(scanner) == '&') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_AND);
+            }
+            return emit_token(current, scanner, TOK_BITAND);
+        };
+        case '|': {
+            if (scan_peek(scanner) == '|') {
+                scan_next(scanner);
+                return emit_token(current, scanner, TOK_OR);
+            }
+            return emit_token(current, scanner, TOK_BITOR);
+        };
+        case ':':
+            return emit_token(current, scanner, TOK_COLON);
+        case '.':
+            return emit_token(current, scanner, TOK_DOT);
+        case '@':
+            return emit_token(current, scanner, TOK_AT);
+        case '"': {
+            while (scan_has_next(scanner)) {
+                c = scan_next(scanner);
+                if (c == '\\') {
+                    scan_next(scanner);
+                    continue;
+                }
+                if (c == '"') {
+                    return emit_token(current, scanner, TOK_STRING);
+                }
+            }
+            return emit_token_err(&current, cstr("mismatched string quotes"));
+        };
+    }
+    if (c >= '0' && c <= '9') {
+        *scanner = current;
+        return emit_token_number(scanner);
+    }
 
-    // At this point we have an error, find the next newline.
-    scan_skip_line(scanner);
+    // TODO: keywords & literals
+    // Basic literals.
+    // TOK_SYMBOL,
+
+    // // Keywords.
+    // TOK_LET,
+    // TOK_SET,
+    // TOK_FUN,
+    // TOK_STRUCT,
+    // TOK_IF,
+    // TOK_MATCH,
+    // TOK_CASE,
+    // TOK_WHILE,
+    // TOK_CONTINUE,
+    // TOK_BREAK,
+    // TOK_RETURN,
+    // TOK_NIL,
+    // TOK_TRUE,
+    // TOK_FALSE,
+
+    // At this point we have an error, skip until we find whitespace again.
+    scan_skip_until_valid(scanner);
     return emit_token_err(&current, cstr("unexpected character"));
 }
 
@@ -281,12 +538,13 @@ process_file(Str path) {
     Token tok = {0};
     while (tok.type != TOK_EOF) {
         tok = scan_token(&scanner);
-        println("%s:%d:%d:%s %s", path, tok.line, tok.col, token_str[tok.type],
-                tok.val);
+        eprintln("%s:%d:%d:%s %s", path, tok.line, tok.col, token_str[tok.type],
+                 tok.val);
     }
     // while (true) {
     //     Token tok = scan_token(&scanner);
-    //     println("%s:%d:%d:%s %s", path, tok.line, tok.col, token_str[tok.type],
+    //     println("%s:%d:%d:%s %s", path, tok.line, tok.col,
+    //     token_str[tok.type],
     //             tok.val);
     //     if (tok.type == TOK_EOF) break;
     // }