VSinerva/compiler-course
Archived
1
0
Fork 0
Code Activity

Change parser tests to use tokenizer

Browse source
This commit is contained in:
Vili Sinervä 2025-01-31 13:03:38 +02:00
parent 8ea712df0b
commit afbfdc7caa
No known key found for this signature in database
GPG key ID: DF8FEAF54EFAC996
1 changed files with 38 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

225
src/compiler/parser/tests.rs
View file

@ -1,13 +1,5 @@
use super::*;
use crate::compiler::token::CodeLocation;
fn int_tok(text: &str) -> Token {
Token::new(
text,
TokenType::Integer,
CodeLocation::new(usize::MAX, usize::MAX),
)
}
use crate::compiler::tokenizer::tokenize;
macro_rules! int_ast {
($x:expr) => {
@ -15,28 +7,12 @@ macro_rules! int_ast {
};
}
fn id_tok(text: &str) -> Token {
Token::new(
text,
TokenType::Identifier,
CodeLocation::new(usize::MAX, usize::MAX),
)
}
macro_rules! id_ast {
($x:expr) => {
Box::new(Identifier($x))
};
}
fn punc_tok(text: &str) -> Token {
Token::new(
text,
TokenType::Punctuation,
CodeLocation::new(usize::MAX, usize::MAX),
)
}
macro_rules! bin_ast {
($x:expr, $y:expr, $z:expr) => {
Box::new(BinaryOp($x, $y, $z))
@ -58,68 +34,48 @@ fn test_empty() {
#[test]
#[should_panic]
fn test_invalid_start() {
parse(&vec![int_tok("1"), int_tok("2"), id_tok("+"), int_tok("3")]);
parse(&tokenize("1 2 + 3"));
}
#[test]
#[should_panic]
fn test_invalid_middle() {
parse(&vec![
int_tok("1"),
id_tok("+"),
int_tok("2"),
int_tok("2"),
id_tok("+"),
int_tok("3"),
]);
parse(&tokenize("1 + 2 2 + 3"));
}
#[test]
#[should_panic]
fn test_invalid_end() {
parse(&vec![
int_tok("1"),
id_tok("+"),
int_tok("2"),
int_tok("2"),
id_tok("+"),
int_tok("3"),
]);
parse(&tokenize("1 + 2 3"));
}
#[test]
fn test_binary_op_basic() {
let result = parse(&vec![int_tok("1"), id_tok("+"), int_tok("23")]);
let result = parse(&tokenize("1 + 23"));
assert_eq!(result, BinaryOp(int_ast!(1), "+", int_ast!(23)));
let result = parse(&vec![int_tok("4"), id_tok("-"), int_tok("56")]);
let result = parse(&tokenize("4 - 56"));
assert_eq!(result, BinaryOp(int_ast!(4), "-", int_ast!(56)));
let result = parse(&vec![int_tok("1"), id_tok("*"), int_tok("2")]);
let result = parse(&tokenize("1 * 2"));
assert_eq!(result, BinaryOp(int_ast!(1), "*", int_ast!(2)));
let result = parse(&vec![int_tok("1"), id_tok("/"), int_tok("2")]);
let result = parse(&tokenize("1 / 2"));
assert_eq!(result, BinaryOp(int_ast!(1), "/", int_ast!(2)));
}
#[test]
fn test_binary_op_identifier() {
let result = parse(&vec![id_tok("a"), id_tok("+"), int_tok("1")]);
let result = parse(&tokenize("a + 1"));
assert_eq!(result, BinaryOp(id_ast!("a"), "+", int_ast!(1)));
let result = parse(&vec![int_tok("1"), id_tok("-"), id_tok("a")]);
let result = parse(&tokenize("1 - a"));
assert_eq!(result, BinaryOp(int_ast!(1), "-", id_ast!("a")));
}
#[test]
fn test_binary_op_multiple() {
let result = parse(&vec![
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("-"),
int_tok("3"),
]);
let result = parse(&tokenize("1 + 2 - 3"));
assert_eq!(
result,
BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "-", int_ast!(3))
@ -128,25 +84,13 @@ fn test_binary_op_multiple() {
#[test]
fn test_binary_op_precedence() {
let result = parse(&vec![
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("*"),
int_tok("3"),
]);
let result = parse(&tokenize("1 + 2 * 3"));
assert_eq!(
result,
BinaryOp(int_ast!(1), "+", bin_ast!(int_ast!(2), "*", int_ast!(3)),)
);
let result = parse(&vec![
int_tok("1"),
id_tok("-"),
int_tok("2"),
id_tok("/"),
int_tok("3"),
]);
let result = parse(&tokenize("1 - 2 / 3"));
assert_eq!(
result,
BinaryOp(int_ast!(1), "-", bin_ast!(int_ast!(2), "/", int_ast!(3)),)
@ -155,15 +99,7 @@ fn test_binary_op_precedence() {
#[test]
fn test_parenthesized() {
let result = parse(&vec![
id_tok("("),
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok(")"),
id_tok("*"),
int_tok("3"),
]);
let result = parse(&tokenize("(1+2)*3"));
assert_eq!(
result,
BinaryOp(bin_ast!(int_ast!(1), "+", int_ast!(2)), "*", int_ast!(3),)
@ -172,46 +108,32 @@ fn test_parenthesized() {
#[test]
fn test_parenthesized_nested() {
let result = parse(&vec![
id_tok("("),
id_tok("("),
int_tok("1"),
id_tok("-"),
int_tok("2"),
id_tok(")"),
id_tok(")"),
id_tok("/"),
int_tok("3"),
]);
let result = parse(&tokenize("((1 - 2))/3"));
assert_eq!(
result,
BinaryOp(bin_ast!(int_ast!(1), "-", int_ast!(2)), "/", int_ast!(3),)
);
let result = parse(&tokenize("((1 + 2)*3) / 4"));
assert_eq!(
result,
BinaryOp(
bin_ast!(bin_ast!(int_ast!(1), "+", int_ast!(2)), "*", int_ast!(3)),
"/",
int_ast!(4)
)
);
}
#[test]
#[should_panic]
fn test_parenthesized_mismatched() {
parse(&vec![
id_tok("("),
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("*"),
int_tok("3"),
]);
parse(&tokenize("(1+2*3"));
}
#[test]
fn test_if_then() {
let result = parse(&vec![
id_tok("if"),
int_tok("1"),
id_tok("+"),
int_tok("2"),
id_tok("then"),
int_tok("3"),
]);
let result = parse(&tokenize("if 1 + 2 then 3"));
assert_eq!(
result,
Conditional(bin_ast!(int_ast!(1), "+", int_ast!(2)), int_ast!(3), None,)
@ -220,18 +142,7 @@ fn test_if_then() {
#[test]
fn test_if_then_else() {
let result = parse(&vec![
id_tok("if"),
id_tok("a"),
id_tok("then"),
id_tok("b"),
id_tok("+"),
id_tok("c"),
id_tok("else"),
int_tok("1"),
id_tok("*"),
int_tok("2"),
]);
let result = parse(&tokenize("if a then b + c else 1 * 2"));
assert_eq!(
result,
Conditional(
@ -244,16 +155,7 @@ fn test_if_then_else() {
#[test]
fn test_if_then_else_embedded() {
let result = parse(&vec![
int_tok("1"),
id_tok("+"),
id_tok("if"),
id_tok("true"),
id_tok("then"),
int_tok("2"),
id_tok("else"),
int_tok("3"),
]);
let result = parse(&tokenize("1 + if true then 2 else 3"));
assert_eq!(
result,
BinaryOp(
@ -266,19 +168,7 @@ fn test_if_then_else_embedded() {
#[test]
fn test_if_then_else_nested() {
let result = parse(&vec![
id_tok("if"),
id_tok("true"),
id_tok("then"),
id_tok("if"),
id_tok("false"),
id_tok("then"),
int_tok("1"),
id_tok("else"),
int_tok("2"),
id_tok("else"),
int_tok("3"),
]);
let result = parse(&tokenize("if true then if false then 1 else 2 else 3"));
assert_eq!(
result,
Conditional(
@ -296,34 +186,18 @@ fn test_if_then_else_nested() {
#[test]
#[should_panic]
fn test_if_no_then() {
parse(&vec![id_tok("if"), id_tok("true")]);
parse(&tokenize("if true"));
}
#[test]
fn test_func_basic() {
let result = parse(&vec![
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(","),
id_tok("b"),
punc_tok(")"),
]);
let result = parse(&tokenize("f(a, b)"));
assert_eq!(
result,
FunCall("f", vec![Identifier("a"), Identifier("b"),])
);
let result = parse(&vec![
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(","),
int_tok("1"),
id_tok("+"),
int_tok("2"),
punc_tok(")"),
]);
let result = parse(&tokenize("f(a, 1 + 2)"));
assert_eq!(
result,
FunCall(
@ -332,20 +206,13 @@ fn test_func_basic() {
)
);
let result = parse(&vec![id_tok("f"), punc_tok("("), punc_tok(")")]);
let result = parse(&tokenize("f()"));
assert_eq!(result, FunCall("f", vec![]));
}
#[test]
fn test_func_embedded() {
let result = parse(&vec![
int_tok("1"),
id_tok("+"),
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(")"),
]);
let result = parse(&tokenize("1 + f(a)"));
assert_eq!(
result,
BinaryOp(
@ -358,17 +225,7 @@ fn test_func_embedded() {
#[test]
fn test_func_nested() {
let result = parse(&vec![
id_tok("f"),
punc_tok("("),
id_tok("a"),
punc_tok(","),
id_tok("g"),
punc_tok("("),
id_tok("b"),
punc_tok(")"),
punc_tok(")"),
]);
let result = parse(&tokenize("f(a, g(b))"));
assert_eq!(
result,
FunCall(
@ -381,17 +238,11 @@ fn test_func_nested() {
#[test]
#[should_panic]
fn test_func_missing_comma() {
parse(&vec![
id_tok("f"),
punc_tok("("),
id_tok("a"),
id_tok("b"),
punc_tok(")"),
]);
parse(&tokenize("f(a b)"));
}
#[test]
#[should_panic]
fn test_func_missing_close() {
parse(&vec![id_tok("f"), punc_tok("("), id_tok("a")]);
parse(&tokenize("f(a"));
}