VSinerva/compiler-course
Archived
1
0
Fork 0
Code Activity

Update tokenizer tests

Browse source
This commit is contained in:
Vili Sinervä 2025-01-21 18:59:47 +02:00
parent 28a8ae69be
commit 65d437b324
No known key found for this signature in database
GPG key ID: DF8FEAF54EFAC996
1 changed files with 38 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/compiler/tokenizer.rs
View file

@ -79,10 +79,41 @@ mod tests {
);
}
#[test]
fn test_tokenize_code_location() {
let result = tokenize("if 3\n while");
use TokenType::*;
assert_eq!(
result,
vec!(
Token::new("if", Identifier, CodeLocation::new(1, 1)),
Token::new("3", Integer, CodeLocation::new(1, 4)),
Token::new("while", Identifier, CodeLocation::new(2, 3)),
)
);
assert_ne!(
result,
vec!(
Token::new("if", Identifier, CodeLocation::new(1, 1)),
Token::new("3", Integer, CodeLocation::new(1, 5)),
Token::new("while", Identifier, CodeLocation::new(2, 3)),
)
);
assert_ne!(
result,
vec!(
Token::new("if", Identifier, CodeLocation::new(1, 1)),
Token::new("3", Integer, CodeLocation::new(1, 4)),
Token::new("while", Identifier, CodeLocation::new(1, 3)),
)
);
}
#[test]
fn test_tokenize_comment() {
let loc = CodeLocation::new(usize::MAX, usize::MAX);
let result = tokenize("if 3 \n\n\\\\Comment\n#Another\n\twhile");
let result = tokenize("if 3 \n\n//Comment\n#Another\n\twhile");
use TokenType::*;
assert_eq!(
@ -166,4 +197,10 @@ mod tests {
)
);
}
#[test]
#[should_panic]
fn test_tokenize_wrong_token() {
tokenize("if 3\n while %");
}
}