From afbfdc7caaf8131141cf95f53fe8c888c2d869d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vili=20Sinerv=C3=A4?= Date: Fri, 31 Jan 2025 13:03:38 +0200 Subject: [PATCH] Change parser tests to use tokenizer --- src/compiler/parser/tests.rs | 225 ++++++----------------------------- 1 file changed, 38 insertions(+), 187 deletions(-) diff --git a/src/compiler/parser/tests.rs b/src/compiler/parser/tests.rs index 441ea51..b74f83d 100644 --- a/src/compiler/parser/tests.rs +++ b/src/compiler/parser/tests.rs @@ -1,13 +1,5 @@ use super::*; -use crate::compiler::token::CodeLocation; - -fn int_tok(text: &str) -> Token { - Token::new( - text, - TokenType::Integer, - CodeLocation::new(usize::MAX, usize::MAX), - ) -} +use crate::compiler::tokenizer::tokenize; macro_rules! int_ast { ($x:expr) => { @@ -15,28 +7,12 @@ macro_rules! int_ast { }; } -fn id_tok(text: &str) -> Token { - Token::new( - text, - TokenType::Identifier, - CodeLocation::new(usize::MAX, usize::MAX), - ) -} - macro_rules! id_ast { ($x:expr) => { Box::new(Identifier($x)) }; } -fn punc_tok(text: &str) -> Token { - Token::new( - text, - TokenType::Punctuation, - CodeLocation::new(usize::MAX, usize::MAX), - ) -} - macro_rules! bin_ast { ($x:expr, $y:expr, $z:expr) => { Box::new(BinaryOp($x, $y, $z)) @@ -58,68 +34,48 @@ fn test_empty() { #[test] #[should_panic] fn test_invalid_start() { - parse(&vec![int_tok("1"), int_tok("2"), id_tok("+"), int_tok("3")]); + parse(&tokenize("1 2 + 3")); } #[test] #[should_panic] fn test_invalid_middle() { - parse(&vec![ - int_tok("1"), - id_tok("+"), - int_tok("2"), - int_tok("2"), - id_tok("+"), - int_tok("3"), - ]); + parse(&tokenize("1 + 2 2 + 3")); } #[test] #[should_panic] fn test_invalid_end() { - parse(&vec![ - int_tok("1"), - id_tok("+"), - int_tok("2"), - int_tok("2"), - id_tok("+"), - int_tok("3"), - ]); + parse(&tokenize("1 + 2 3")); } #[test] fn test_binary_op_basic() { - let result = parse(&vec![int_tok("1"), id_tok("+"), int_tok("23")]); + let result = parse(&tokenize("1 + 23")); assert_eq!(result, BinaryOp(int_ast!(1), "+", int_ast!(23))); - let result = parse(&vec![int_tok("4"), id_tok("-"), int_tok("56")]); + let result = parse(&tokenize("4 - 56")); assert_eq!(result, BinaryOp(int_ast!(4), "-", int_ast!(56))); - let result = parse(&vec![int_tok("1"), id_tok("*"), int_tok("2")]); + let result = parse(&tokenize("1 * 2")); assert_eq!(result, BinaryOp(int_ast!(1), "*", int_ast!(2))); - let result = parse(&vec![int_tok("1"), id_tok("/"), int_tok("2")]); + let result = parse(&tokenize("1 / 2")); assert_eq!(result, BinaryOp(int_ast!(1), "/", int_ast!(2))); } #[test] fn test_binary_op_identifier() { - let result = parse(&vec![id_tok("a"), id_tok("+"), int_tok("1")]); + let result = parse(&tokenize("a + 1")); assert_eq!(result, BinaryOp(id_ast!("a"), "+", int_ast!(1))); - let result = parse(&vec![int_tok("1"), id_tok("-"), id_tok("a")]); + let result = parse(&tokenize("1 - a")); assert_eq!(result, BinaryOp(int_ast!(1), "-", id_ast!("a"))); } #[test] fn test_binary_op_multiple() { - let result = parse(&vec![ - int_tok("1"), - id_tok("+"), - int_tok("2"), - id_tok("-"), - int_tok("3"), - ]); + let result = parse(&tokenize("1 + 2 - 3")); assert_eq!( result, BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "-", int_ast!(3)) @@ -128,25 +84,13 @@ fn test_binary_op_multiple() { #[test] fn test_binary_op_precedence() { - let result = parse(&vec![ - int_tok("1"), - id_tok("+"), - int_tok("2"), - id_tok("*"), - int_tok("3"), - ]); + let result = parse(&tokenize("1 + 2 * 3")); assert_eq!( result, BinaryOp(int_ast!(1), "+", bin_ast!(int_ast!(2), "*", int_ast!(3)),) ); - let result = parse(&vec![ - int_tok("1"), - id_tok("-"), - int_tok("2"), - id_tok("/"), - int_tok("3"), - ]); + let result = parse(&tokenize("1 - 2 / 3")); assert_eq!( result, BinaryOp(int_ast!(1), "-", bin_ast!(int_ast!(2), "/", int_ast!(3)),) @@ -155,15 +99,7 @@ fn test_binary_op_precedence() { #[test] fn test_parenthesized() { - let result = parse(&vec![ - id_tok("("), - int_tok("1"), - id_tok("+"), - int_tok("2"), - id_tok(")"), - id_tok("*"), - int_tok("3"), - ]); + let result = parse(&tokenize("(1+2)*3")); assert_eq!( result, BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "*", int_ast!(3),) @@ -172,46 +108,32 @@ fn test_parenthesized() { #[test] fn test_parenthesized_nested() { - let result = parse(&vec![ - id_tok("("), - id_tok("("), - int_tok("1"), - id_tok("-"), - int_tok("2"), - id_tok(")"), - id_tok(")"), - id_tok("/"), - int_tok("3"), - ]); + let result = parse(&tokenize("((1 - 2))/3")); assert_eq!( result, BinaryOp(bin_ast!(int_ast!(1), "-", int_ast!(2)), "/", int_ast!(3),) ); + + let result = parse(&tokenize("((1 + 2)*3) / 4")); + assert_eq!( + result, + BinaryOp( + bin_ast!(bin_ast!(int_ast!(1), "+", int_ast!(2)), "*", int_ast!(3)), + "/", + int_ast!(4) + ) + ); } #[test] #[should_panic] fn test_parenthesized_mismatched() { - parse(&vec![ - id_tok("("), - int_tok("1"), - id_tok("+"), - int_tok("2"), - id_tok("*"), - int_tok("3"), - ]); + parse(&tokenize("(1+2*3")); } #[test] fn test_if_then() { - let result = parse(&vec![ - id_tok("if"), - int_tok("1"), - id_tok("+"), - int_tok("2"), - id_tok("then"), - int_tok("3"), - ]); + let result = parse(&tokenize("if 1 + 2 then 3")); assert_eq!( result, Conditional(bin_ast!(int_ast!(1), "+", int_ast!(2)), int_ast!(3), None,) @@ -220,18 +142,7 @@ fn test_if_then() { #[test] fn test_if_then_else() { - let result = parse(&vec![ - id_tok("if"), - id_tok("a"), - id_tok("then"), - id_tok("b"), - id_tok("+"), - id_tok("c"), - id_tok("else"), - int_tok("1"), - id_tok("*"), - int_tok("2"), - ]); + let result = parse(&tokenize("if a then b + c else 1 * 2")); assert_eq!( result, Conditional( @@ -244,16 +155,7 @@ fn test_if_then_else() { #[test] fn test_if_then_else_embedded() { - let result = parse(&vec![ - int_tok("1"), - id_tok("+"), - id_tok("if"), - id_tok("true"), - id_tok("then"), - int_tok("2"), - id_tok("else"), - int_tok("3"), - ]); + let result = parse(&tokenize("1 + if true then 2 else 3")); assert_eq!( result, BinaryOp( @@ -266,19 +168,7 @@ fn test_if_then_else_embedded() { #[test] fn test_if_then_else_nested() { - let result = parse(&vec![ - id_tok("if"), - id_tok("true"), - id_tok("then"), - id_tok("if"), - id_tok("false"), - id_tok("then"), - int_tok("1"), - id_tok("else"), - int_tok("2"), - id_tok("else"), - int_tok("3"), - ]); + let result = parse(&tokenize("if true then if false then 1 else 2 else 3")); assert_eq!( result, Conditional( @@ -296,34 +186,18 @@ fn test_if_then_else_nested() { #[test] #[should_panic] fn test_if_no_then() { - parse(&vec![id_tok("if"), id_tok("true")]); + parse(&tokenize("if true")); } #[test] fn test_func_basic() { - let result = parse(&vec![ - id_tok("f"), - punc_tok("("), - id_tok("a"), - punc_tok(","), - id_tok("b"), - punc_tok(")"), - ]); + let result = parse(&tokenize("f(a, b)")); assert_eq!( result, FunCall("f", vec![Identifier("a"), Identifier("b"),]) ); - let result = parse(&vec![ - id_tok("f"), - punc_tok("("), - id_tok("a"), - punc_tok(","), - int_tok("1"), - id_tok("+"), - int_tok("2"), - punc_tok(")"), - ]); + let result = parse(&tokenize("f(a, 1 + 2)")); assert_eq!( result, FunCall( @@ -332,20 +206,13 @@ fn test_func_basic() { ) ); - let result = parse(&vec![id_tok("f"), punc_tok("("), punc_tok(")")]); + let result = parse(&tokenize("f()")); assert_eq!(result, FunCall("f", vec![])); } #[test] fn test_func_embedded() { - let result = parse(&vec![ - int_tok("1"), - id_tok("+"), - id_tok("f"), - punc_tok("("), - id_tok("a"), - punc_tok(")"), - ]); + let result = parse(&tokenize("1 + f(a)")); assert_eq!( result, BinaryOp( @@ -358,17 +225,7 @@ fn test_func_embedded() { #[test] fn test_func_nested() { - let result = parse(&vec![ - id_tok("f"), - punc_tok("("), - id_tok("a"), - punc_tok(","), - id_tok("g"), - punc_tok("("), - id_tok("b"), - punc_tok(")"), - punc_tok(")"), - ]); + let result = parse(&tokenize("f(a, g(b))")); assert_eq!( result, FunCall( @@ -381,17 +238,11 @@ fn test_func_nested() { #[test] #[should_panic] fn test_func_missing_comma() { - parse(&vec![ - id_tok("f"), - punc_tok("("), - id_tok("a"), - id_tok("b"), - punc_tok(")"), - ]); + parse(&tokenize("f(a b)")); } #[test] #[should_panic] fn test_func_missing_close() { - parse(&vec![id_tok("f"), punc_tok("("), id_tok("a")]); + parse(&tokenize("f(a")); }