tokeniser: refactor to store Module directly in Tokeniser

We hereby avoid making extra copies of the PathBuf.

- Also updated tests to match the new API
This commit is contained in:
2025-06-28 23:13:44 +01:00
parent a65dca6c5c
commit d15e00c272
2 changed files with 54 additions and 69 deletions
+7 -6
View File
@@ -3,7 +3,7 @@
use common::prelude::Register;
use crate::{
context::AssemblerContext,
model::module::Module,
source::{
opcode::Opcode,
token::{Token, TokenType},
@@ -11,20 +11,21 @@ use crate::{
tokeniser::Tokeniser,
},
};
use std::path::PathBuf;
use std::{path::PathBuf, sync::Arc};
/// Helper function to create a tokenizer from source text
fn create_tokenizer_from_source(source: &str) -> Tokeniser {
let data = source.as_bytes().to_vec();
let path = PathBuf::from("test.dsa");
Tokeniser::from_data(data, path)
let module = Module::new(path).expect("Cannot create module!");
Tokeniser::from_data(source.as_bytes().to_vec(), Arc::new(module))
}
/// Helper function to tokenize source and return tokens
fn tokenize_source(source: &str) -> Result<Vec<Token>, crate::error::AssembleError> {
let tokenizer = create_tokenizer_from_source(source);
let context = AssemblerContext::new();
tokenizer.tokenise(&context)
tokenizer.tokenise()
}
/// Helper function to extract token types from a token vector