use super::lex; use crate::logger::messages::Message; use crate::parsing::lexer::lex_tokens::TokenType; use std::assert_matches::assert_matches; macro_rules! lex_token_test { ( $a: ident, $b: expr, $c: expr) => { #[test] fn $a() { let tokens = lex($b, &mut |_message, _span| { std::panic::panic_any(_message.stringify()); }); assert_eq!(tokens.len(), 2); assert_eq!(tokens[0].token_type, $c); assert_eq!(tokens[0].span.start, 0); assert_eq!(tokens[0].span.end, $b.chars().count()); assert_eq!(tokens[1].token_type, TokenType::EndOfFile); } }; } macro_rules! lex_identifier_test { ( $a: ident, $b: expr) => { #[test] fn $a() { let tokens = lex($b, &mut |_message, _span| { std::panic::panic_any(_message.stringify()); }); assert_eq!(tokens.len(), 2); assert_eq!(tokens[0].token_type, TokenType::Identifier($b.to_string())); assert_eq!(tokens[0].span.start, 0); assert_eq!(tokens[0].span.end, $b.chars().count()); assert_eq!(tokens[1].token_type, TokenType::EndOfFile); } }; } macro_rules! lex_integer_test { ( $a: ident, $b: expr, $c: expr) => { #[test] fn $a() { let tokens = lex($b, &mut |_message, _span| { std::panic::panic_any(_message.stringify()); }); assert_eq!(tokens.len(), 2); assert_eq!(tokens[0].token_type, TokenType::IntegerLiteral($c)); assert_eq!(tokens[0].span.start, 0); assert_eq!(tokens[0].span.end, $b.chars().count()); assert_eq!(tokens[1].token_type, TokenType::EndOfFile); } }; } macro_rules! lex_float_test { ( $a: ident, $b: expr, $c: expr) => { #[test] fn $a() { let tokens = lex($b, &mut |_message, _span| { std::panic::panic_any(_message.stringify()); }); assert_eq!(tokens.len(), 2); assert_eq!(tokens[0].token_type, TokenType::FloatLiteral($c)); assert_eq!(tokens[0].span.start, 0); assert_eq!(tokens[0].span.end, $b.chars().count()); assert_eq!(tokens[1].token_type, TokenType::EndOfFile); } }; } macro_rules! lex_string_test { ( $a: ident, $b: expr, $c: expr) => { #[test] fn $a() { let tokens = lex($b, &mut |_message, _span| { std::panic::panic_any(_message.stringify()); }); assert_eq!(tokens.len(), 2); assert_eq!( tokens[0].token_type, TokenType::StringLiteral($c.to_string()) ); assert_eq!(tokens[0].span.start, 0); assert_eq!(tokens[0].span.end, $b.chars().count()); assert_eq!(tokens[1].token_type, TokenType::EndOfFile); } }; } lex_token_test!(lex_space, " ", TokenType::WhiteSpace); lex_token_test!(lex_tab, "\t", TokenType::WhiteSpace); lex_token_test!(lex_return_line, "\r", TokenType::WhiteSpace); lex_token_test!(lex_newline, "\n", TokenType::WhiteSpace); lex_token_test!(lex_equals, "=", TokenType::Equals); lex_token_test!(lex_equals_equals, "==", TokenType::EqualsEquals); lex_token_test!(lex_plus, "+", TokenType::Plus); lex_token_test!(lex_plus_plus, "++", TokenType::PlusPlus); lex_token_test!(lex_plus_equals, "+=", TokenType::PlusEquals); lex_token_test!(lex_minus, "-", TokenType::Minus); lex_token_test!(lex_minus_minus, "--", TokenType::MinusMinus); lex_token_test!(lex_minus_equals, "-=", TokenType::MinusEquals); lex_token_test!(lex_star, "*", TokenType::Star); lex_token_test!(lex_star_equals, "*=", TokenType::StarEquals); lex_token_test!(lex_star_star, "**", TokenType::StarStar); lex_token_test!(lex_star_star_equals, "**=", TokenType::StarStarEquals); lex_token_test!(lex_slash, "/", TokenType::Slash); lex_token_test!(lex_slash_equals, "/=", TokenType::SlashEquals); lex_token_test!(lex_percent, "%", TokenType::Percent); lex_token_test!(lex_percent_equals, "%=", TokenType::PercentEquals); lex_token_test!(lex_exclamation_mark, "!", TokenType::ExclamationMark); lex_token_test!(lex_not_equals, "!=", TokenType::NotEquals); lex_token_test!(lex_not_is_keyword, "!is", TokenType::NotIsKeyword); lex_token_test!(lex_vert_line, "|", TokenType::VerticalLine); lex_token_test!(lex_vert_line_equals, "|=", TokenType::LineEquals); lex_token_test!(lex_line_line, "||", TokenType::LineLine); lex_token_test!(lex_ampersand, "&", TokenType::Ampersand); lex_token_test!(lex_ampersand_equals, "&=", TokenType::AmpersandEquals); lex_token_test!(lex_ampersand_ampersand, "&&", TokenType::AmpersandAmpersand); lex_token_test!(lex_less_than, "<", TokenType::LessThan); lex_token_test!(lex_less_than_equals, "<=", TokenType::LessThanEquals); lex_token_test!(lex_left_left, "<<", TokenType::LeftLeft); lex_token_test!(lex_left_left_equals, "<<=", TokenType::LeftLeftEquals); lex_token_test!(lex_greater_than, ">", TokenType::GreaterThan); lex_token_test!(lex_greater_than_equals, ">=", TokenType::GreaterThanEquals); lex_token_test!(lex_right_right, ">>", TokenType::RightRight); lex_token_test!(lex_right_right_equals, ">>=", TokenType::RightRightEquals); lex_token_test!(lex_right_right_right, ">>>", TokenType::RightRightRight); lex_token_test!( lex_right_right_right_equals, ">>>=", TokenType::RightRightRightEquals ); lex_token_test!(lex_tilde, "~", TokenType::Tilde); lex_token_test!(lex_at_symbol, "@", TokenType::AtSymbol); lex_token_test!(lex_colon, ":", TokenType::Colon); lex_token_test!(lex_coloncolon, "::", TokenType::ColonColon); lex_token_test!(lex_and_keyword, "and", TokenType::AndKeyword); lex_token_test!(lex_abstract_keyword, "abstract", TokenType::AbstractKeyword); lex_token_test!(lex_auto_keyword, "auto", TokenType::AutoKeyword); lex_token_test!(lex_bool_keyword, "bool", TokenType::BoolKeyword); lex_token_test!(lex_break_keyword, "break", TokenType::BreakKeyword); lex_token_test!(lex_case_keyword, "case", TokenType::CaseKeyword); lex_token_test!(lex_cast_keyword, "cast", TokenType::CastKeyword); lex_token_test!(lex_catch_keyword, "catch", TokenType::CatchKeyword); lex_token_test!(lex_class_keyword, "class", TokenType::ClassKeyword); lex_token_test!(lex_const_keyword, "const", TokenType::ConstKeyword); lex_token_test!(lex_continue_keyword, "continue", TokenType::ContinueKeyword); lex_token_test!(lex_default_keyword, "default", TokenType::DefaultKeyword); lex_token_test!(lex_do_keyword, "do", TokenType::DoKeyword); lex_token_test!(lex_double_keyword, "double", TokenType::DoubleKeyword); lex_token_test!(lex_else_keyword, "else", TokenType::ElseKeyword); lex_token_test!(lex_enum_keyword, "enum", TokenType::EnumKeyword); lex_token_test!(lex_explicit_keyword, "explicit", TokenType::ExplicitKeyword); lex_token_test!(lex_external_keyword, "external", TokenType::ExternalKeyword); lex_token_test!(lex_false_keyword, "false", TokenType::FalseKeyword); lex_token_test!(lex_final_keyword, "final", TokenType::FinalKeyword); lex_token_test!(lex_float_keyword, "float", TokenType::FloatKeyword); lex_token_test!(lex_for_keyword, "for", TokenType::ForKeyword); lex_token_test!(lex_from_keyword, "from", TokenType::FromKeyword); lex_token_test!(lex_funcdef_keyword, "funcdef", TokenType::FuncDefKeyword); lex_token_test!(lex_function_keyword, "function", TokenType::FunctionKeyword); lex_token_test!(lex_get_keyword, "get", TokenType::GetKeyword); lex_token_test!(lex_if_keyword, "if", TokenType::IfKeyword); lex_token_test!(lex_import_keyword, "import", TokenType::ImportKeyword); lex_token_test!(lex_in_keyword, "in", TokenType::InKeyword); lex_token_test!(lex_inout_keyword, "inout", TokenType::InOutKeyword); lex_token_test!(lex_int_keyword, "int", TokenType::IntKeyword); lex_token_test!( lex_interface_keyword, "interface", TokenType::InterfaceKeyword ); lex_token_test!(lex_int8_keyword, "int8", TokenType::Int8Keyword); lex_token_test!(lex_int16_keyword, "int16", TokenType::Int16Keyword); lex_token_test!(lex_int32_keyword, "int32", TokenType::Int32Keyword); lex_token_test!(lex_int64_keyword, "int64", TokenType::Int64Keyword); lex_token_test!(lex_is_keyword, "is", TokenType::IsKeyword); lex_token_test!(lex_mixin_keyword, "mixin", TokenType::MixinKeyword); lex_token_test!( lex_namespace_keyword, "namespace", TokenType::NamespaceKeyword ); lex_token_test!(lex_not_keyword, "not", TokenType::NotKeyword); lex_token_test!(lex_null_keyword, "null", TokenType::NullKeyword); lex_token_test!(lex_or_keyword, "or", TokenType::OrKeyword); lex_token_test!(lex_out_keyword, "out", TokenType::OutKeyword); lex_token_test!(lex_override_keyword, "override", TokenType::OverrideKeyword); lex_token_test!(lex_private_keyword, "private", TokenType::PrivateKeyword); lex_token_test!(lex_property_keyword, "property", TokenType::PropertyKeyword); lex_token_test!( lex_protected_keyword, "protected", TokenType::ProtectedKeyword ); lex_token_test!(lex_return_keyword, "return", TokenType::ReturnKeyword); lex_token_test!(lex_set_keyword, "set", TokenType::SetKeyword); lex_token_test!(lex_shared_keyword, "shared", TokenType::SharedKeyword); lex_token_test!(lex_super_keyword, "super", TokenType::SuperKeyword); lex_token_test!(lex_switch_keyword, "switch", TokenType::SwitchKeyword); lex_token_test!(lex_this_keyword, "this", TokenType::ThisKeyword); lex_token_test!(lex_true_keyword, "true", TokenType::TrueKeyword); lex_token_test!(lex_try_keyword, "try", TokenType::TryKeyword); lex_token_test!(lex_typedef_keyword, "typedef", TokenType::TypeDefKeyword); lex_token_test!(lex_uint_keyword, "uint", TokenType::UintKeyword); lex_token_test!(lex_uint8_keyword, "uint8", TokenType::Uint8Keyword); lex_token_test!(lex_uint16_keyword, "uint16", TokenType::Uint16Keyword); lex_token_test!(lex_uint32_keyword, "uint32", TokenType::Uint32Keyword); lex_token_test!(lex_void_keyword, "void", TokenType::VoidKeyword); lex_token_test!(lex_while_keyword, "while", TokenType::WhileKeyword); lex_token_test!(lex_xor_keyword, "xor", TokenType::XorKeyword); lex_identifier_test!(lex_basic_identifier_foo, "foo"); lex_identifier_test!(lex_basic_identifier_foobar, "foobar"); lex_integer_test!(lex_zero, "0", 0); lex_integer_test!(lex_one_two_three_four, "1234", 1234); lex_integer_test!(lex_specific_one_two_three_four, "0d1234", 1234); lex_integer_test!(lex_decimal_with_underline, "123_456", 123456); lex_integer_test!(lex_specific_decimal_with_underline, "0D123_456", 123456); lex_integer_test!(lex_hexadecimal_0f, "0X0F", 15); lex_integer_test!(lex_hexadecimal_ff, "0xff", 255); lex_integer_test!(lex_hexadecimal_ff_ff, "0xff_ff", 65535); lex_integer_test!(lex_octal_112, "0o112", 74); lex_integer_test!(lex_binary_1110, "0b1110", 14); lex_integer_test!(lex_binary_01110, "0b01110", 14); lex_float_test!(lex_zero_float, "0.0", 0.0); lex_float_test!(lex_half, "0.5", 0.5); lex_float_test!(lex_point_0_5, "0.05", 0.05); lex_float_test!(lex_half_with_exponent, "0.5e10", 0.5e10); lex_string_test!(lex_simple_string, "\"foo\"", "foo"); lex_string_test!(lex_simple_string_single_quote, "\'foo\'", "foo"); lex_string_test!(lex_string_with_escape, "\"fo\\\"o\"", "fo\"o"); lex_string_test!(lex_string_with_new_line, "\"fo\\no\"", "fo\no"); lex_string_test!(lex_heredoc_string, "\"\"\"foo\"\"\"", "foo"); lex_string_test!(lex_heredoc_string_with_quote, "\"\"\"fo\"o\"\"\"", "fo\"o"); #[test] fn lex_two_identifier() { let tokens = lex("foo bar", &mut |_message, _span| { std::panic::panic_any(_message.stringify()); }); assert_eq!(tokens.len(), 4); assert_eq!( tokens[0].token_type, TokenType::Identifier("foo".to_string()) ); assert_eq!(tokens[0].span.start, 0); assert_eq!(tokens[0].span.end, 3); assert_eq!(tokens[1].token_type, TokenType::WhiteSpace); assert_eq!( tokens[2].token_type, TokenType::Identifier("bar".to_string()) ); assert_eq!(tokens[2].span.start, 4); assert_eq!(tokens[2].span.end, 7); assert_eq!(tokens[3].token_type, TokenType::EndOfFile); } #[test] fn lex_multiple_tokens_with_not_is() { let tokens = lex("a !is b", &mut |_message, _span| { std::panic::panic_any(_message.stringify()); }); assert_eq!(tokens.len(), 6); assert_eq!(tokens[0].token_type, TokenType::Identifier("a".to_string())); assert_eq!(tokens[0].span.start, 0); assert_eq!(tokens[0].span.end, 1); assert_eq!(tokens[1].token_type, TokenType::WhiteSpace); assert_eq!(tokens[1].span.start, 1); assert_eq!(tokens[1].span.end, 2); assert_eq!(tokens[2].token_type, TokenType::NotIsKeyword); assert_eq!(tokens[2].span.start, 2); assert_eq!(tokens[2].span.end, 5); assert_eq!(tokens[3].token_type, TokenType::WhiteSpace); assert_eq!(tokens[3].span.start, 5); assert_eq!(tokens[3].span.end, 6); assert_eq!(tokens[4].token_type, TokenType::Identifier("b".to_string())); assert_eq!(tokens[4].span.start, 6); assert_eq!(tokens[4].span.end, 7); assert_eq!(tokens[5].token_type, TokenType::EndOfFile); assert_eq!(tokens[5].span.start, 7); assert_eq!(tokens[5].span.end, 7); } #[test] fn lex_invalid_character_at_first_position() { let mut reached = false; lex("\x08", &mut |message, span| { reached = true; assert_matches!(message, Message::UnexpectedCharacter('\x08')); assert_eq!(span.start, 0); assert_eq!(span.end, 1); }); assert!(reached); } #[test] fn lex_invalid_character_at_other_position() { let mut reached = false; lex(" \x08", &mut |message, span| { reached = true; assert_matches!(message, Message::UnexpectedCharacter('\x08')); assert_eq!(span.start, 2); assert_eq!(span.end, 3); }); assert!(reached); } #[test] fn lex_unclosed_string_literal() { let mut reached = false; lex("\" ", &mut |message, span| { reached = true; assert_matches!(message, Message::UnclosedStringLiteral); assert_eq!(span.start, 5); assert_eq!(span.end, 6); }); assert!(reached); }