242 lines
10 KiB
Rust
242 lines
10 KiB
Rust
use super::lex_tokens::LexToken;
|
|
use super::lexer::lex;
|
|
|
|
macro_rules! lex_token_test {
|
|
( $a: ident, $b: expr, $c: expr) => {
|
|
#[test]
|
|
fn $a() {
|
|
let tokens = lex($b);
|
|
assert_eq!(tokens.len(), 2);
|
|
assert_eq!(tokens[0], $c);
|
|
assert_eq!(tokens[1], LexToken::EndOfFile);
|
|
}
|
|
};
|
|
}
|
|
|
|
macro_rules! lex_identifier_test {
|
|
( $a: ident, $b: expr) => {
|
|
#[test]
|
|
fn $a() {
|
|
let tokens = lex($b);
|
|
assert_eq!(tokens.len(), 2);
|
|
assert_eq!(tokens[0], LexToken::Identifier($b.to_string()));
|
|
assert_eq!(tokens[1], LexToken::EndOfFile);
|
|
}
|
|
};
|
|
}
|
|
|
|
macro_rules! lex_integer_test {
|
|
( $a: ident, $b: expr, $c: expr) => {
|
|
#[test]
|
|
fn $a() {
|
|
let tokens = lex($b);
|
|
assert_eq!(tokens.len(), 2);
|
|
assert_eq!(tokens[0], LexToken::IntegerLiteral($c));
|
|
assert_eq!(tokens[1], LexToken::EndOfFile);
|
|
}
|
|
};
|
|
}
|
|
|
|
macro_rules! lex_float_test {
|
|
( $a: ident, $b: expr, $c: expr) => {
|
|
#[test]
|
|
fn $a() {
|
|
let tokens = lex($b);
|
|
assert_eq!(tokens.len(), 2);
|
|
assert_eq!(tokens[0], LexToken::FloatLiteral($c));
|
|
assert_eq!(tokens[1], LexToken::EndOfFile);
|
|
}
|
|
};
|
|
}
|
|
|
|
macro_rules! lex_string_test {
|
|
( $a: ident, $b: expr, $c: expr) => {
|
|
#[test]
|
|
fn $a() {
|
|
let tokens = lex($b);
|
|
assert_eq!(tokens.len(), 2);
|
|
assert_eq!(tokens[0], LexToken::StringLiteral($c.to_string()));
|
|
assert_eq!(tokens[1], LexToken::EndOfFile);
|
|
}
|
|
};
|
|
}
|
|
|
|
lex_token_test!(lex_space, " ", LexToken::WhiteSpace);
|
|
lex_token_test!(lex_tab, "\t", LexToken::WhiteSpace);
|
|
lex_token_test!(lex_return_line, "\r", LexToken::WhiteSpace);
|
|
lex_token_test!(lex_newline, "\n", LexToken::WhiteSpace);
|
|
|
|
lex_token_test!(lex_equals, "=", LexToken::Equals);
|
|
lex_token_test!(lex_equals_equals, "==", LexToken::EqualsEquals);
|
|
lex_token_test!(lex_plus, "+", LexToken::Plus);
|
|
lex_token_test!(lex_plus_plus, "++", LexToken::PlusPlus);
|
|
lex_token_test!(lex_plus_equals, "+=", LexToken::PlusEquals);
|
|
lex_token_test!(lex_minus, "-", LexToken::Minus);
|
|
lex_token_test!(lex_minus_minus, "--", LexToken::MinusMinus);
|
|
lex_token_test!(lex_minus_equals, "-=", LexToken::MinusEquals);
|
|
lex_token_test!(lex_star, "*", LexToken::Star);
|
|
lex_token_test!(lex_star_equals, "*=", LexToken::StarEquals);
|
|
lex_token_test!(lex_star_star, "**", LexToken::StarStar);
|
|
lex_token_test!(lex_star_star_equals, "**=", LexToken::StarStarEquals);
|
|
lex_token_test!(lex_slash, "/", LexToken::Slash);
|
|
lex_token_test!(lex_slash_equals, "/=", LexToken::SlashEquals);
|
|
lex_token_test!(lex_percent, "%", LexToken::Percent);
|
|
lex_token_test!(lex_percent_equals, "%=", LexToken::PercentEquals);
|
|
|
|
lex_token_test!(lex_exclamation_mark, "!", LexToken::ExclamationMark);
|
|
lex_token_test!(lex_not_equals, "!=", LexToken::NotEquals);
|
|
lex_token_test!(lex_not_is_keyword, "!is", LexToken::NotIsKeyword);
|
|
|
|
lex_token_test!(lex_vert_line, "|", LexToken::VerticalLine);
|
|
lex_token_test!(lex_vert_line_equals, "|=", LexToken::LineEquals);
|
|
lex_token_test!(lex_line_line, "||", LexToken::LineLine);
|
|
|
|
lex_token_test!(lex_ampersand, "&", LexToken::Ampersand);
|
|
lex_token_test!(lex_ampersand_equals, "&=", LexToken::AmpersandEquals);
|
|
lex_token_test!(lex_ampersand_ampersand, "&&", LexToken::AmpersandAmpersand);
|
|
lex_token_test!(lex_less_than, "<", LexToken::LessThan);
|
|
lex_token_test!(lex_less_than_equals, "<=", LexToken::LessThanEquals);
|
|
lex_token_test!(lex_left_left, "<<", LexToken::LeftLeft);
|
|
lex_token_test!(lex_left_left_equals, "<<=", LexToken::LeftLeftEquals);
|
|
|
|
lex_token_test!(lex_greater_than, ">", LexToken::GreaterThan);
|
|
lex_token_test!(lex_greater_than_equals, ">=", LexToken::GreaterThanEquals);
|
|
lex_token_test!(lex_right_right, ">>", LexToken::RightRight);
|
|
lex_token_test!(lex_right_right_equals, ">>=", LexToken::RightRightEquals);
|
|
lex_token_test!(lex_right_right_right, ">>>", LexToken::RightRightRight);
|
|
lex_token_test!(
|
|
lex_right_right_right_equals,
|
|
">>>=",
|
|
LexToken::RightRightRightEquals
|
|
);
|
|
|
|
lex_token_test!(lex_tilde, "~", LexToken::Tilde);
|
|
lex_token_test!(lex_at_symbol, "@", LexToken::AtSymbol);
|
|
|
|
lex_token_test!(lex_and_keyword, "and", LexToken::AndKeyword);
|
|
lex_token_test!(lex_abstract_keyword, "abstract", LexToken::AbstractKeyword);
|
|
lex_token_test!(lex_auto_keyword, "auto", LexToken::AutoKeyword);
|
|
lex_token_test!(lex_bool_keyword, "bool", LexToken::BoolKeyword);
|
|
lex_token_test!(lex_break_keyword, "break", LexToken::BreakKeyword);
|
|
lex_token_test!(lex_case_keyword, "case", LexToken::CaseKeyword);
|
|
lex_token_test!(lex_cast_keyword, "cast", LexToken::CastKeyword);
|
|
lex_token_test!(lex_catch_keyword, "catch", LexToken::CatchKeyword);
|
|
lex_token_test!(lex_class_keyword, "class", LexToken::ClassKeyword);
|
|
lex_token_test!(lex_const_keyword, "const", LexToken::ConstKeyword);
|
|
lex_token_test!(lex_continue_keyword, "continue", LexToken::ContinueKeyword);
|
|
lex_token_test!(lex_default_keyword, "default", LexToken::DefaultKeyword);
|
|
lex_token_test!(lex_do_keyword, "do", LexToken::DoKeyword);
|
|
lex_token_test!(lex_double_keyword, "double", LexToken::DoubleKeyword);
|
|
lex_token_test!(lex_else_keyword, "else", LexToken::ElseKeyword);
|
|
lex_token_test!(lex_enum_keyword, "enum", LexToken::EnumKeyword);
|
|
lex_token_test!(lex_explicit_keyword, "explicit", LexToken::ExplicitKeyword);
|
|
lex_token_test!(lex_external_keyword, "external", LexToken::ExternalKeyword);
|
|
lex_token_test!(lex_false_keyword, "false", LexToken::FalseKeyword);
|
|
lex_token_test!(lex_final_keyword, "final", LexToken::FinalKeyword);
|
|
lex_token_test!(lex_float_keyword, "float", LexToken::FloatKeyword);
|
|
lex_token_test!(lex_for_keyword, "for", LexToken::ForKeyword);
|
|
lex_token_test!(lex_from_keyword, "from", LexToken::FromKeyword);
|
|
lex_token_test!(lex_funcdef_keyword, "funcdef", LexToken::FuncDefKeyword);
|
|
lex_token_test!(lex_function_keyword, "function", LexToken::FunctionKeyword);
|
|
lex_token_test!(lex_get_keyword, "get", LexToken::GetKeyword);
|
|
lex_token_test!(lex_if_keyword, "if", LexToken::IfKeyword);
|
|
lex_token_test!(lex_import_keyword, "import", LexToken::ImportKeyword);
|
|
lex_token_test!(lex_in_keyword, "in", LexToken::InKeyword);
|
|
lex_token_test!(lex_inout_keyword, "inout", LexToken::InOutKeyword);
|
|
lex_token_test!(lex_int_keyword, "int", LexToken::IntKeyword);
|
|
lex_token_test!(
|
|
lex_interface_keyword,
|
|
"interface",
|
|
LexToken::InterfaceKeyword
|
|
);
|
|
lex_token_test!(lex_int8_keyword, "int8", LexToken::Int8Keyword);
|
|
lex_token_test!(lex_int16_keyword, "int16", LexToken::Int16Keyword);
|
|
lex_token_test!(lex_int32_keyword, "int32", LexToken::Int32Keyword);
|
|
lex_token_test!(lex_int64_keyword, "int64", LexToken::Int64Keyword);
|
|
lex_token_test!(lex_is_keyword, "is", LexToken::IsKeyword);
|
|
lex_token_test!(lex_mixin_keyword, "mixin", LexToken::MixinKeyword);
|
|
lex_token_test!(
|
|
lex_namespace_keyword,
|
|
"namespace",
|
|
LexToken::NamespaceKeyword
|
|
);
|
|
lex_token_test!(lex_not_keyword, "not", LexToken::NotKeyword);
|
|
lex_token_test!(lex_null_keyword, "null", LexToken::NullKeyword);
|
|
lex_token_test!(lex_or_keyword, "or", LexToken::OrKeyword);
|
|
lex_token_test!(lex_out_keyword, "out", LexToken::OutKeyword);
|
|
lex_token_test!(lex_override_keyword, "override", LexToken::OverrideKeyword);
|
|
lex_token_test!(lex_private_keyword, "private", LexToken::PrivateKeyword);
|
|
lex_token_test!(lex_property_keyword, "property", LexToken::PropertyKeyword);
|
|
lex_token_test!(
|
|
lex_protected_keyword,
|
|
"protected",
|
|
LexToken::ProtectedKeyword
|
|
);
|
|
|
|
lex_token_test!(lex_return_keyword, "return", LexToken::ReturnKeyword);
|
|
lex_token_test!(lex_set_keyword, "set", LexToken::SetKeyword);
|
|
lex_token_test!(lex_shared_keyword, "shared", LexToken::SharedKeyword);
|
|
lex_token_test!(lex_super_keyword, "super", LexToken::SuperKeyword);
|
|
lex_token_test!(lex_switch_keyword, "switch", LexToken::SwitchKeyword);
|
|
lex_token_test!(lex_this_keyword, "this", LexToken::ThisKeyword);
|
|
lex_token_test!(lex_true_keyword, "true", LexToken::TrueKeyword);
|
|
lex_token_test!(lex_try_keyword, "try", LexToken::TryKeyword);
|
|
lex_token_test!(lex_typedef_keyword, "typedef", LexToken::TypeDefKeyword);
|
|
lex_token_test!(lex_uint_keyword, "uint", LexToken::UintKeyword);
|
|
lex_token_test!(lex_uint8_keyword, "uint8", LexToken::Uint8Keyword);
|
|
lex_token_test!(lex_uint16_keyword, "uint16", LexToken::Uint16Keyword);
|
|
lex_token_test!(lex_uint32_keyword, "uint32", LexToken::Uint32Keyword);
|
|
|
|
lex_token_test!(lex_void_keyword, "void", LexToken::VoidKeyword);
|
|
lex_token_test!(lex_while_keyword, "while", LexToken::WhileKeyword);
|
|
lex_token_test!(lex_xor_keyword, "xor", LexToken::XorKeyword);
|
|
|
|
lex_identifier_test!(lex_basic_identifier_foo, "foo");
|
|
lex_identifier_test!(lex_basic_identifier_foobar, "foobar");
|
|
|
|
lex_integer_test!(lex_zero, "0", 0);
|
|
lex_integer_test!(lex_one_two_three_four, "1234", 1234);
|
|
lex_integer_test!(lex_specific_one_two_three_four, "0d1234", 1234);
|
|
lex_integer_test!(lex_decimal_with_underline, "123_456", 123456);
|
|
lex_integer_test!(lex_specific_decimal_with_underline, "0D123_456", 123456);
|
|
lex_integer_test!(lex_hexadecimal_0f, "0X0F", 15);
|
|
lex_integer_test!(lex_hexadecimal_ff, "0xff", 255);
|
|
lex_integer_test!(lex_hexadecimal_ff_ff, "0xff_ff", 65535);
|
|
lex_integer_test!(lex_octal_112, "0o112", 74);
|
|
lex_integer_test!(lex_binary_1110, "0b1110", 14);
|
|
lex_integer_test!(lex_binary_01110, "0b01110", 14);
|
|
|
|
lex_float_test!(lex_zero_float, "0.0", 0.0);
|
|
lex_float_test!(lex_half, "0.5", 0.5);
|
|
lex_float_test!(lex_point_0_5, "0.05", 0.05);
|
|
lex_float_test!(lex_half_with_exponent, "0.5e10", 0.5e10);
|
|
|
|
lex_string_test!(lex_simple_string, "\"foo\"", "foo");
|
|
lex_string_test!(lex_simple_string_single_quote, "\'foo\'", "foo");
|
|
lex_string_test!(lex_string_with_escape, "\"fo\\\"o\"", "fo\"o");
|
|
lex_string_test!(lex_string_with_new_line, "\"fo\\no\"", "fo\no");
|
|
lex_string_test!(lex_heredoc_string, "\"\"\"foo\"\"\"", "foo");
|
|
lex_string_test!(lex_heredoc_string_with_quote, "\"\"\"fo\"o\"\"\"", "fo\"o");
|
|
|
|
#[test]
|
|
fn lex_two_identifier() {
|
|
let tokens = lex("foo bar");
|
|
assert_eq!(tokens.len(), 4);
|
|
assert_eq!(tokens[0], LexToken::Identifier("foo".to_string()));
|
|
assert_eq!(tokens[1], LexToken::WhiteSpace);
|
|
assert_eq!(tokens[2], LexToken::Identifier("bar".to_string()));
|
|
assert_eq!(tokens[3], LexToken::EndOfFile);
|
|
}
|
|
|
|
#[test]
|
|
fn lex_multiple_tokens_with_not_is() {
|
|
let tokens = lex("a !is b");
|
|
assert_eq!(tokens.len(), 6);
|
|
assert_eq!(tokens[0], LexToken::Identifier("a".to_string()));
|
|
assert_eq!(tokens[1], LexToken::WhiteSpace);
|
|
assert_eq!(tokens[2], LexToken::NotIsKeyword);
|
|
assert_eq!(tokens[3], LexToken::WhiteSpace);
|
|
assert_eq!(tokens[4], LexToken::Identifier("b".to_string()));
|
|
assert_eq!(tokens[5], LexToken::EndOfFile);
|
|
}
|