changes

dsa/assembler_old/src/assembler/model.rs

@@ -186,6 +186,7 @@ pub enum Token {
     Register(Register),
     Immediate(u32),
     StringLit(String),
+    CharLit(char),
     Opcode(Opcode),
 }
 
@@ -196,6 +197,7 @@ impl fmt::Display for Token {
             Self::Register(register) => write!(f, "{register}",),
             Self::Immediate(immediate) => write!(f, "{immediate}",),
             Self::StringLit(string_lit) => write!(f, "{string_lit}",),
+            Self::CharLit(char_lit) => write!(f, "{char_lit}",),
             Self::Opcode(opcode) => write!(f, "{opcode}",),
         }
     }
@@ -207,6 +209,7 @@ pub enum TokenType {
     Register,
     Immediate,
     StringLit,
+    CharLit,
     Opcode,
 }
 
@@ -218,6 +221,7 @@ impl TokenType {
             Token::Register(_) => Self::Register,
             Token::Immediate(_) => Self::Immediate,
             Token::StringLit(_) => Self::StringLit,
+            Token::CharLit(_) => Self::CharLit,
             Token::Opcode(_) => Self::Opcode,
         }
     }

dsa/assembler_old/src/assemblerv2/lexer.rs

@@ -0,0 +1,199 @@
+use crate::assembler::{AssembleError, Token};
+
+pub struct Lexer {
+    src: Vec<u8>,
+    module_id: u64,
+}
+
+impl Lexer {
+    pub fn new(src: String, module_id: u64) -> Self {
+        Self {
+            src: src.into_bytes().into_iter().rev().collect(),
+            module_id,
+        }
+    }
+
+    pub fn lexer(&mut self) -> Result<Vec<Token>, AssembleError> {
+        let mut tokens = Vec::new();
+
+        while !self.src.is_empty() {
+            if self.src.ends_with(b"//") {
+                self.src.pop();
+                self.src.pop();
+                self.parse_comment();
+                continue;
+            }
+
+            if self.src.ends_with(b"/*") {
+                self.src.pop();
+                self.src.pop();
+                self.parse_comment_multiline();
+                continue;
+            }
+
+            if self.src.ends_with(b"\"") {
+                self.src.pop();
+                tokens.push(Token::StringLit(self.parse_string_literal()));
+            }
+
+            if self.src.ends_with(b"'") {
+                self.src.pop();
+                tokens.push(Token::CharLit(self.parse_char_literal()));
+            }
+
+            if self.src.last().unwrap().is_ascii_digit() {
+                tokens.push(Token::Immediate(self.parse_number()));
+            }
+        }
+
+        Ok(tokens)
+    }
+
+    fn parse_number(&mut self) -> u32 {
+        match self.src.last_chunk::<2>() {
+            Some(b"0x") => {
+                self.src.pop();
+                self.src.pop();
+                let mut n = 0u32;
+                while let Some(&b) = self.src.last() {
+                    if let Some(digit) = (b as char).to_digit(16) {
+                        self.src.pop();
+                        n = n * 16 + digit;
+                    } else {
+                        break;
+                    }
+                }
+                n
+            }
+            Some(b"0b") => {
+                self.src.pop();
+                self.src.pop();
+                let mut n = 0u32;
+                while let Some(&b) = self.src.last() {
+                    if b == b'0' || b == b'1' {
+                        self.src.pop();
+                        n = n * 2 + (b - b'0') as u32;
+                    } else {
+                        break;
+                    }
+                }
+                n
+            }
+            Some(b"0o") => {
+                self.src.pop();
+                self.src.pop();
+                let mut n = 0u32;
+                while let Some(&b) = self.src.last() {
+                    if matches!(b, b'0'..=b'7') {
+                        self.src.pop();
+                        n = n * 8 + (b - b'0') as u32;
+                    } else {
+                        break;
+                    }
+                }
+                n
+            }
+            _ if self.src.last().is_some_and(|b| b.is_ascii_digit()) => {
+                let mut n = 0u32;
+                while let Some(&b) = self.src.last() {
+                    if b.is_ascii_digit() {
+                        self.src.pop();
+                        n = n * 10 + (b - b'0') as u32;
+                    } else {
+                        break;
+                    }
+                }
+                n
+            }
+            None if let Some(x) = self.src.last()
+                && x.is_ascii_digit() =>
+            {
+                let x = *x;
+                self.src.pop();
+                (x - b'0') as u32
+            }
+            _ => panic!("Invalid syntax"),
+        }
+    }
+
+    // returns nothing as the assembler discards comments
+    fn parse_comment(&mut self) {
+        while !self.src.is_empty() && self.src.ends_with(b"\n") {
+            self.src.pop();
+        }
+    }
+
+    fn parse_comment_multiline(&mut self) {
+        while !self.src.is_empty() && self.src.ends_with(b"*/") {
+            self.src.pop();
+        }
+    }
+
+    fn parse_string_literal(&mut self) -> String {
+        let mut result = String::new();
+        while let Some(ch) = self.src.pop()
+            && self.src.ends_with(b"\"")
+        {
+            if ch == b'\\' {
+                let escaped = self
+                    .src
+                    .pop()
+                    .expect("A file should never end with a backslash!");
+                result.push(match escaped {
+                    b'n' => '\n',
+                    b't' => '\t',
+                    b'r' => '\r',
+                    b'"' => '\"',
+                    b'\\' => '\\',
+                    _ => escaped as char,
+                });
+
+                continue;
+            }
+
+            result.push(ch as char);
+        }
+
+        result
+    }
+
+    fn parse_char_literal(&mut self) -> char {
+        let ch = self
+            .src
+            .pop()
+            .expect("Unexpected EOF while parsing char literal");
+
+        if ch == b'\\' {
+            let escaped = self
+                .src
+                .pop()
+                .expect("A file should never end with a backslash!");
+
+            assert!(
+                self.src
+                    .pop()
+                    .expect("Unexpected EOF while parsing char literal")
+                    != b'\'',
+                "unterminated char literal"
+            );
+
+            match escaped {
+                b'n' => '\n',
+                b't' => '\t',
+                b'r' => '\r',
+                b'"' => '\"',
+                b'\\' => '\\',
+                _ => escaped as char,
+            }
+        } else {
+            assert!(
+                self.src
+                    .pop()
+                    .expect("Unexpected EOF while parsing char literal")
+                    != b'\'',
+                "unterminated char literal"
+            );
+            ch as char
+        }
+    }
+}

dsa/assembler_old/src/assemblerv2/mod.rs

@@ -0,0 +1,2 @@
+pub mod lexer;
+pub mod parser;

dsa/assembler_old/src/lib.rs

@@ -13,6 +13,7 @@
 )]
 
 pub mod assembler;
+pub mod assemblerv2;
 // mod util;
 
 pub mod prelude {