VSinerva/compiler-course
Archived
1
0
Fork 0
Code Activity

Test for and fix bug in tokenizer RE: comments

Browse source
This commit is contained in:
Vili Sinervä 2025-02-26 18:53:39 +02:00
parent 051c264a23
commit a8cef15331
No known key found for this signature in database
GPG key ID: DF8FEAF54EFAC996
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/compiler/tokenizer.rs
View file

@ -43,6 +43,7 @@ pub fn tokenize(code: &str) -> Vec<Token> {
valid_token = true;
pos += token.end();
break;
}
}
@ -62,7 +63,6 @@ pub fn tokenize(code: &str) -> Vec<Token> {
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_tokenize_basic() {
let loc = CodeLocation::new(usize::MAX, usize::MAX);
@ -113,7 +113,7 @@ mod tests {
#[test]
fn test_tokenize_comment() {
let loc = CodeLocation::new(usize::MAX, usize::MAX);
let result = tokenize("if 3 \n\n//Comment\n#Another\n\twhile");
let result = tokenize("if 3 \n\n//Comment\n#Another\n\twhile //Comment2");
use TokenType::*;
assert_eq!(