added some documentation and started on compiler for custom language (not C) based on previous prototypes. pretty broken state rn.

compiler/src/lexer.rs

@@ -4,12 +4,15 @@ use std::str::Chars;
 #[derive(Debug, PartialEq, Clone)]
 pub enum Token {
     // Keywords
+    Fn,
+    Let,
     If,
     Else,
     Loop,
     Break,
     Return,
     Continue,
+    Include,
 
     // Identifiers and literals
     Identifier(String),
@@ -24,7 +27,7 @@ pub enum Token {
     Semicolon,  // ;
     Colon,      // :
     Comma,      // ,
-    Pipe,       // |
+    // Pipe,       // |
 
     // Operators
     Plus,         // +
@@ -39,6 +42,7 @@ pub enum Token {
     LessEqual,    // <=
     Greater,      // >
     GreaterEqual, // >=
+    RightArrow,   // ->
 
     // Special
     Eof,
@@ -47,7 +51,10 @@ pub enum Token {
 impl Token {
     pub fn tt(&self) -> &str {
         match self {
+            Token::Include => "Include",
+            Token::Fn => "Fn",
             Token::If => "If",
+            Token::Let => "Let",
             Token::Else => "Else",
             Token::Loop => "Loop",
             Token::Break => "Break",
@@ -63,7 +70,8 @@ impl Token {
             Token::Semicolon => "Semicolon",
             Token::Colon => "Colon",
             Token::Comma => "Comma",
-            Token::Pipe => "Pipe",
+            Token::RightArrow => "RightArrow",
+            // Token::Pipe => "Pipe",
             Token::Plus => "Plus",
             Token::Minus => "Minus",
             Token::Star => "Star",
@@ -168,11 +176,17 @@ impl<'a> Lexer<'a> {
             Some(';') => Token::Semicolon,
             Some(':') => Token::Colon,
             Some(',') => Token::Comma,
-            Some('|') => Token::Pipe,
+            // Some('|') => Token::Pipe,
             Some('+') => Token::Plus,
-            Some('-') => Token::Minus,
             Some('*') => Token::Star,
             Some('/') => Token::Slash,
+            Some('-') => {
+                if self.match_next('>') {
+                    Token::RightArrow
+                } else {
+                    Token::Minus
+                }
+            }
             Some('!') => {
                 if self.match_next('=') {
                     Token::BangEqual
@@ -218,12 +232,14 @@ impl<'a> Lexer<'a> {
                     let mut ident = c.to_string();
                     ident.push_str(&self.read_identifier());
                     match ident.as_str() {
+                        "fn" => Token::Fn,
                         "if" => Token::If,
                         "else" => Token::Else,
                         "loop" => Token::Loop,
                         "break" => Token::Break,
                         "return" => Token::Return,
                         "continue" => Token::Continue,
+                        "include" => Token::Include,
                         _ => Token::Identifier(ident),
                     }
                 } else if c.is_ascii_digit() {
@@ -331,7 +347,7 @@ mod tests {
         assert_eq!(lexer.next_token(), Token::Colon);
         assert_eq!(lexer.next_token(), Token::Identifier("Func".to_string()));
         assert_eq!(lexer.next_token(), Token::Assign);
-        assert_eq!(lexer.next_token(), Token::Pipe);
+        // assert_eq!(lexer.next_token(), Token::Pipe);
         assert_eq!(lexer.next_token(), Token::Identifier("x".to_string()));
         assert_eq!(lexer.next_token(), Token::Colon);
         assert_eq!(lexer.next_token(), Token::Identifier("U32".to_string()));