assembler: start refactoring/rewriting tokeniser

assembler/src/source/token.rs

@@ -2,7 +2,12 @@
 //! easier to build from scratch and edit his code than it would be to try and wrangle it
 //! into shape.
 
-use crate::source::source_info::SourceInfo;
+use crate::source::{
+    source_info::SourceInfo,
+    token_info::{
+        DirectiveToken, InstructionToken, LabelToken, RegisterToken, SymbolToken,
+    },
+};
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum TokenType {
@@ -29,34 +34,9 @@ pub enum TokenType {
 #[derive(Debug)]
 pub struct Token {
     /// The type of the token.
-    token_type: TokenType,
+    pub token_type: TokenType,
     /// Where in the source code is this [`Token`]?
-    source_info: SourceInfo,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct SymbolToken {
-    pub name: String,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct LabelToken {
-    pub name: String,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct DirectiveToken {
-    pub directive: String,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct RegisterToken {
-    pub name: String,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct InstructionToken {
-    pub mnemonic: String,
+    pub source_info: SourceInfo,
 }
 
 impl Token {