From 65d437b3248f7045d998d242480ce7bd525c7369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vili=20Sinerv=C3=A4?= Date: Tue, 21 Jan 2025 18:59:47 +0200 Subject: [PATCH] Update tokenizer tests --- src/compiler/tokenizer.rs | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/compiler/tokenizer.rs b/src/compiler/tokenizer.rs index 98cdeeb..c525084 100644 --- a/src/compiler/tokenizer.rs +++ b/src/compiler/tokenizer.rs @@ -79,10 +79,41 @@ mod tests { ); } + #[test] + fn test_tokenize_code_location() { + let result = tokenize("if 3\n while"); + + use TokenType::*; + assert_eq!( + result, + vec!( + Token::new("if", Identifier, CodeLocation::new(1, 1)), + Token::new("3", Integer, CodeLocation::new(1, 4)), + Token::new("while", Identifier, CodeLocation::new(2, 3)), + ) + ); + assert_ne!( + result, + vec!( + Token::new("if", Identifier, CodeLocation::new(1, 1)), + Token::new("3", Integer, CodeLocation::new(1, 5)), + Token::new("while", Identifier, CodeLocation::new(2, 3)), + ) + ); + assert_ne!( + result, + vec!( + Token::new("if", Identifier, CodeLocation::new(1, 1)), + Token::new("3", Integer, CodeLocation::new(1, 4)), + Token::new("while", Identifier, CodeLocation::new(1, 3)), + ) + ); + } + #[test] fn test_tokenize_comment() { let loc = CodeLocation::new(usize::MAX, usize::MAX); - let result = tokenize("if 3 \n\n\\\\Comment\n#Another\n\twhile"); + let result = tokenize("if 3 \n\n//Comment\n#Another\n\twhile"); use TokenType::*; assert_eq!( @@ -166,4 +197,10 @@ mod tests { ) ); } + + #[test] + #[should_panic] + fn test_tokenize_wrong_token() { + tokenize("if 3\n while %"); + } }