VSinerva/compiler-course
Archived
1
0
Fork 0
Code Activity

Change parser tests to use tokenizer

Browse source
This commit is contained in:
Vili Sinervä 2025-01-31 13:03:38 +02:00
parent 8ea712df0b
commit afbfdc7caa
No known key found for this signature in database
GPG key ID: DF8FEAF54EFAC996
1 changed files with 38 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

225
src/compiler/parser/tests.rs
View file

@ -1,13 +1,5 @@
use super::*; use super::*;
use crate::compiler::token::CodeLocation; use crate::compiler::tokenizer::tokenize;
fn int_tok(text: &str) -> Token {
Token::new(
text,
TokenType::Integer,
CodeLocation::new(usize::MAX, usize::MAX),
)
}
macro_rules! int_ast { macro_rules! int_ast {
($x:expr) => { ($x:expr) => {
@ -15,28 +7,12 @@ macro_rules! int_ast {
}; };
} }
fn id_tok(text: &str) -> Token {
Token::new(
text,
TokenType::Identifier,
CodeLocation::new(usize::MAX, usize::MAX),
)
}
macro_rules! id_ast { macro_rules! id_ast {
($x:expr) => { ($x:expr) => {
Box::new(Identifier($x)) Box::new(Identifier($x))
}; };
} }
fn punc_tok(text: &str) -> Token {
Token::new(
text,
TokenType::Punctuation,
CodeLocation::new(usize::MAX, usize::MAX),
)
}
macro_rules! bin_ast { macro_rules! bin_ast {
($x:expr, $y:expr, $z:expr) => { ($x:expr, $y:expr, $z:expr) => {
Box::new(BinaryOp($x, $y, $z)) Box::new(BinaryOp($x, $y, $z))
@ -58,68 +34,48 @@ fn test_empty() {
#[test] #[test]
#[should_panic] #[should_panic]
fn test_invalid_start() { fn test_invalid_start() {
parse(&vec![int_tok("1"), int_tok("2"), id_tok("+"), int_tok("3")]); parse(&tokenize("1 2 + 3"));
} }
#[test] #[test]
#[should_panic] #[should_panic]
fn test_invalid_middle() { fn test_invalid_middle() {
parse(&vec![ parse(&tokenize("1 + 2 2 + 3"));
int_tok("1"),
id_tok("+"),
int_tok("2"),
int_tok("2"),
id_tok("+"),
int_tok("3"),
]);
} }
#[test] #[test]
#[should_panic] #[should_panic]
fn test_invalid_end() { fn test_invalid_end() {
parse(&vec![ parse(&tokenize("1 + 2 3"));
int_tok("1"),
id_tok("+"),
int_tok("2"),
int_tok("2"),
id_tok("+"),
int_tok("3"),
]);
} }
#[test] #[test]
fn test_binary_op_basic() { fn test_binary_op_basic() {
let result = parse(&vec![int_tok("1"), id_tok("+"), int_tok("23")]); let result = parse(&tokenize("1 + 23"));
assert_eq!(result, BinaryOp(int_ast!(1), "+", int_ast!(23))); assert_eq!(result, BinaryOp(int_ast!(1), "+", int_ast!(23)));
let result = parse(&vec![int_tok("4"), id_tok("-"), int_tok("56")]); let result = parse(&tokenize("4 - 56"));
assert_eq!(result, BinaryOp(int_ast!(4), "-", int_ast!(56))); assert_eq!(result, BinaryOp(int_ast!(4), "-", int_ast!(56)));
let result = parse(&vec![int_tok("1"), id_tok("*"), int_tok("2")]); let result = parse(&tokenize("1 * 2"));
assert_eq!(result, BinaryOp(int_ast!(1), "*", int_ast!(2))); assert_eq!(result, BinaryOp(int_ast!(1), "*", int_ast!(2)));
let result = parse(&vec![int_tok("1"), id_tok("/"), int_tok("2")]); let result = parse(&tokenize("1 / 2"));
assert_eq!(result, BinaryOp(int_ast!(1), "/", int_ast!(2))); assert_eq!(result, BinaryOp(int_ast!(1), "/", int_ast!(2)));
} }
#[test] #[test]
fn test_binary_op_identifier() { fn test_binary_op_identifier() {
let result = parse(&vec![id_tok("a"), id_tok("+"), int_tok("1")]); let result = parse(&tokenize("a + 1"));
assert_eq!(result, BinaryOp(id_ast!("a"), "+", int_ast!(1))); assert_eq!(result, BinaryOp(id_ast!("a"), "+", int_ast!(1)));
let result = parse(&vec![int_tok("1"), id_tok("-"), id_tok("a")]); let result = parse(&tokenize("1 - a"));
assert_eq!(result, BinaryOp(int_ast!(1), "-", id_ast!("a"))); assert_eq!(result, BinaryOp(int_ast!(1), "-", id_ast!("a")));
} }
#[test] #[test]
fn test_binary_op_multiple() { fn test_binary_op_multiple() {
let result = parse(&vec![ let result = parse(&tokenize("1 + 2 - 3"));
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("-"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "-", int_ast!(3)) BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "-", int_ast!(3))
@ -128,25 +84,13 @@ fn test_binary_op_multiple() {
#[test] #[test]
fn test_binary_op_precedence() { fn test_binary_op_precedence() {
let result = parse(&vec![ let result = parse(&tokenize("1 + 2 * 3"));
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("*"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
BinaryOp(int_ast!(1), "+", bin_ast!(int_ast!(2), "*", int_ast!(3)),) BinaryOp(int_ast!(1), "+", bin_ast!(int_ast!(2), "*", int_ast!(3)),)
); );
let result = parse(&vec![ let result = parse(&tokenize("1 - 2 / 3"));
int_tok("1"),
id_tok("-"),
int_tok("2"),
id_tok("/"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
BinaryOp(int_ast!(1), "-", bin_ast!(int_ast!(2), "/", int_ast!(3)),) BinaryOp(int_ast!(1), "-", bin_ast!(int_ast!(2), "/", int_ast!(3)),)
@ -155,15 +99,7 @@ fn test_binary_op_precedence() {
#[test] #[test]
fn test_parenthesized() { fn test_parenthesized() {
let result = parse(&vec![ let result = parse(&tokenize("(1+2)*3"));
id_tok("("),
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok(")"),
id_tok("*"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "*", int_ast!(3),) BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "*", int_ast!(3),)
@ -172,46 +108,32 @@ fn test_parenthesized() {
#[test] #[test]
fn test_parenthesized_nested() { fn test_parenthesized_nested() {
let result = parse(&vec![ let result = parse(&tokenize("((1 - 2))/3"));
id_tok("("),
id_tok("("),
int_tok("1"),
id_tok("-"),
int_tok("2"),
id_tok(")"),
id_tok(")"),
id_tok("/"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
BinaryOp(bin_ast!(int_ast!(1), "-", int_ast!(2)), "/", int_ast!(3),) BinaryOp(bin_ast!(int_ast!(1), "-", int_ast!(2)), "/", int_ast!(3),)
); );
let result = parse(&tokenize("((1 + 2)*3) / 4"));
assert_eq!(
result,
BinaryOp(
bin_ast!(bin_ast!(int_ast!(1), "+", int_ast!(2)), "*", int_ast!(3)),
"/",
int_ast!(4)
)
);
} }
#[test] #[test]
#[should_panic] #[should_panic]
fn test_parenthesized_mismatched() { fn test_parenthesized_mismatched() {
parse(&vec![ parse(&tokenize("(1+2*3"));
id_tok("("),
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("*"),
int_tok("3"),
]);
} }
#[test] #[test]
fn test_if_then() { fn test_if_then() {
let result = parse(&vec![ let result = parse(&tokenize("if 1 + 2 then 3"));
id_tok("if"),
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("then"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
Conditional(bin_ast!(int_ast!(1), "+", int_ast!(2)), int_ast!(3), None,) Conditional(bin_ast!(int_ast!(1), "+", int_ast!(2)), int_ast!(3), None,)
@ -220,18 +142,7 @@ fn test_if_then() {
#[test] #[test]
fn test_if_then_else() { fn test_if_then_else() {
let result = parse(&vec![ let result = parse(&tokenize("if a then b + c else 1 * 2"));
id_tok("if"),
id_tok("a"),
id_tok("then"),
id_tok("b"),
id_tok("+"),
id_tok("c"),
id_tok("else"),
int_tok("1"),
id_tok("*"),
int_tok("2"),
]);
assert_eq!( assert_eq!(
result, result,
Conditional( Conditional(
@ -244,16 +155,7 @@ fn test_if_then_else() {
#[test] #[test]
fn test_if_then_else_embedded() { fn test_if_then_else_embedded() {
let result = parse(&vec![ let result = parse(&tokenize("1 + if true then 2 else 3"));
int_tok("1"),
id_tok("+"),
id_tok("if"),
id_tok("true"),
id_tok("then"),
int_tok("2"),
id_tok("else"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
BinaryOp( BinaryOp(
@ -266,19 +168,7 @@ fn test_if_then_else_embedded() {
#[test] #[test]
fn test_if_then_else_nested() { fn test_if_then_else_nested() {
let result = parse(&vec![ let result = parse(&tokenize("if true then if false then 1 else 2 else 3"));
id_tok("if"),
id_tok("true"),
id_tok("then"),
id_tok("if"),
id_tok("false"),
id_tok("then"),
int_tok("1"),
id_tok("else"),
int_tok("2"),
id_tok("else"),
int_tok("3"),
]);
assert_eq!( assert_eq!(
result, result,
Conditional( Conditional(
@ -296,34 +186,18 @@ fn test_if_then_else_nested() {
#[test] #[test]
#[should_panic] #[should_panic]
fn test_if_no_then() { fn test_if_no_then() {
parse(&vec![id_tok("if"), id_tok("true")]); parse(&tokenize("if true"));
} }
#[test] #[test]
fn test_func_basic() { fn test_func_basic() {
let result = parse(&vec![ let result = parse(&tokenize("f(a, b)"));
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(","),
id_tok("b"),
punc_tok(")"),
]);
assert_eq!( assert_eq!(
result, result,
FunCall("f", vec![Identifier("a"), Identifier("b"),]) FunCall("f", vec![Identifier("a"), Identifier("b"),])
); );
let result = parse(&vec![ let result = parse(&tokenize("f(a, 1 + 2)"));
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(","),
int_tok("1"),
id_tok("+"),
int_tok("2"),
punc_tok(")"),
]);
assert_eq!( assert_eq!(
result, result,
FunCall( FunCall(
@ -332,20 +206,13 @@ fn test_func_basic() {
) )
); );
let result = parse(&vec![id_tok("f"), punc_tok("("), punc_tok(")")]); let result = parse(&tokenize("f()"));
assert_eq!(result, FunCall("f", vec![])); assert_eq!(result, FunCall("f", vec![]));
} }
#[test] #[test]
fn test_func_embedded() { fn test_func_embedded() {
let result = parse(&vec![ let result = parse(&tokenize("1 + f(a)"));
int_tok("1"),
id_tok("+"),
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(")"),
]);
assert_eq!( assert_eq!(
result, result,
BinaryOp( BinaryOp(
@ -358,17 +225,7 @@ fn test_func_embedded() {
#[test] #[test]
fn test_func_nested() { fn test_func_nested() {
let result = parse(&vec![ let result = parse(&tokenize("f(a, g(b))"));
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(","),
id_tok("g"),
punc_tok("("),
id_tok("b"),
punc_tok(")"),
punc_tok(")"),
]);
assert_eq!( assert_eq!(
result, result,
FunCall( FunCall(
@ -381,17 +238,11 @@ fn test_func_nested() {
#[test] #[test]
#[should_panic] #[should_panic]
fn test_func_missing_comma() { fn test_func_missing_comma() {
parse(&vec![ parse(&tokenize("f(a b)"));
id_tok("f"),
punc_tok("("),
id_tok("a"),
id_tok("b"),
punc_tok(")"),
]);
} }
#[test] #[test]
#[should_panic] #[should_panic]
fn test_func_missing_close() { fn test_func_missing_close() {
parse(&vec![id_tok("f"), punc_tok("("), id_tok("a")]); parse(&tokenize("f(a"));
} }