SeraphScript/src/parsing/lexer/lexer_tests.rs

use super::lex;
use crate::logger::messages::Message;
use crate::parsing::lexer::lex_tokens::TokenType;
use std::assert_matches::assert_matches;

macro_rules! lex_token_test {
    ( $a: ident, $b: expr, $c: expr) => {
        #[test]
        fn $a() {
            let tokens = lex($b, &mut |_message, _span| {
                std::panic::panic_any(_message.stringify());
            });
            assert_eq!(tokens.len(), 2);
            assert_eq!(tokens[0].token_type, $c);
            assert_eq!(tokens[0].span.start, 0);
            assert_eq!(tokens[0].span.end, $b.chars().count());
            assert_eq!(tokens[1].token_type, TokenType::EndOfFile);
        }
    };
}

macro_rules! lex_identifier_test {
    ( $a: ident, $b: expr) => {
        #[test]
        fn $a() {
            let tokens = lex($b, &mut |_message, _span| {
                std::panic::panic_any(_message.stringify());
            });
            assert_eq!(tokens.len(), 2);
            assert_eq!(tokens[0].token_type, TokenType::Identifier($b.to_string()));
            assert_eq!(tokens[0].span.start, 0);
            assert_eq!(tokens[0].span.end, $b.chars().count());
            assert_eq!(tokens[1].token_type, TokenType::EndOfFile);
        }
    };
}

macro_rules! lex_integer_test {
    ( $a: ident, $b: expr, $c: expr) => {
        #[test]
        fn $a() {
            let tokens = lex($b, &mut |_message, _span| {
                std::panic::panic_any(_message.stringify());
            });
            assert_eq!(tokens.len(), 2);
            assert_eq!(tokens[0].token_type, TokenType::IntegerLiteral($c));
            assert_eq!(tokens[0].span.start, 0);
            assert_eq!(tokens[0].span.end, $b.chars().count());
            assert_eq!(tokens[1].token_type, TokenType::EndOfFile);
        }
    };
}

macro_rules! lex_float_test {
    ( $a: ident, $b: expr, $c: expr) => {
        #[test]
        fn $a() {
            let tokens = lex($b, &mut |_message, _span| {
                std::panic::panic_any(_message.stringify());
            });
            assert_eq!(tokens.len(), 2);
            assert_eq!(tokens[0].token_type, TokenType::FloatLiteral($c));
            assert_eq!(tokens[0].span.start, 0);
            assert_eq!(tokens[0].span.end, $b.chars().count());
            assert_eq!(tokens[1].token_type, TokenType::EndOfFile);
        }
    };
}

macro_rules! lex_string_test {
    ( $a: ident, $b: expr, $c: expr) => {
        #[test]
        fn $a() {
            let tokens = lex($b, &mut |_message, _span| {
                std::panic::panic_any(_message.stringify());
            });
            assert_eq!(tokens.len(), 2);
            assert_eq!(
                tokens[0].token_type,
                TokenType::StringLiteral($c.to_string())
            );
            assert_eq!(tokens[0].span.start, 0);
            assert_eq!(tokens[0].span.end, $b.chars().count());
            assert_eq!(tokens[1].token_type, TokenType::EndOfFile);
        }
    };
}

lex_token_test!(lex_space, " ", TokenType::WhiteSpace);
lex_token_test!(lex_tab, "\t", TokenType::WhiteSpace);
lex_token_test!(lex_return_line, "\r", TokenType::WhiteSpace);
lex_token_test!(lex_newline, "\n", TokenType::WhiteSpace);

lex_token_test!(lex_equals, "=", TokenType::Equals);
lex_token_test!(lex_equals_equals, "==", TokenType::EqualsEquals);
lex_token_test!(lex_plus, "+", TokenType::Plus);
lex_token_test!(lex_plus_plus, "++", TokenType::PlusPlus);
lex_token_test!(lex_plus_equals, "+=", TokenType::PlusEquals);
lex_token_test!(lex_minus, "-", TokenType::Minus);
lex_token_test!(lex_minus_minus, "--", TokenType::MinusMinus);
lex_token_test!(lex_minus_equals, "-=", TokenType::MinusEquals);
lex_token_test!(lex_star, "*", TokenType::Star);
lex_token_test!(lex_star_equals, "*=", TokenType::StarEquals);
lex_token_test!(lex_star_star, "**", TokenType::StarStar);
lex_token_test!(lex_star_star_equals, "**=", TokenType::StarStarEquals);
lex_token_test!(lex_slash, "/", TokenType::Slash);
lex_token_test!(lex_slash_equals, "/=", TokenType::SlashEquals);
lex_token_test!(lex_percent, "%", TokenType::Percent);
lex_token_test!(lex_percent_equals, "%=", TokenType::PercentEquals);

lex_token_test!(lex_exclamation_mark, "!", TokenType::ExclamationMark);
lex_token_test!(lex_not_equals, "!=", TokenType::NotEquals);
lex_token_test!(lex_not_is_keyword, "!is", TokenType::NotIsKeyword);

lex_token_test!(lex_vert_line, "|", TokenType::VerticalLine);
lex_token_test!(lex_vert_line_equals, "|=", TokenType::LineEquals);
lex_token_test!(lex_line_line, "||", TokenType::LineLine);

lex_token_test!(lex_ampersand, "&", TokenType::Ampersand);
lex_token_test!(lex_ampersand_equals, "&=", TokenType::AmpersandEquals);
lex_token_test!(lex_ampersand_ampersand, "&&", TokenType::AmpersandAmpersand);
lex_token_test!(lex_less_than, "<", TokenType::LessThan);
lex_token_test!(lex_less_than_equals, "<=", TokenType::LessThanEquals);
lex_token_test!(lex_left_left, "<<", TokenType::LeftLeft);
lex_token_test!(lex_left_left_equals, "<<=", TokenType::LeftLeftEquals);

lex_token_test!(lex_greater_than, ">", TokenType::GreaterThan);
lex_token_test!(lex_greater_than_equals, ">=", TokenType::GreaterThanEquals);
lex_token_test!(lex_right_right, ">>", TokenType::RightRight);
lex_token_test!(lex_right_right_equals, ">>=", TokenType::RightRightEquals);
lex_token_test!(lex_right_right_right, ">>>", TokenType::RightRightRight);
lex_token_test!(
    lex_right_right_right_equals,
    ">>>=",
    TokenType::RightRightRightEquals
);

lex_token_test!(lex_tilde, "~", TokenType::Tilde);
lex_token_test!(lex_at_symbol, "@", TokenType::AtSymbol);

lex_token_test!(lex_colon, ":", TokenType::Colon);
lex_token_test!(lex_coloncolon, "::", TokenType::ColonColon);

lex_token_test!(lex_and_keyword, "and", TokenType::AndKeyword);
lex_token_test!(lex_abstract_keyword, "abstract", TokenType::AbstractKeyword);
lex_token_test!(lex_auto_keyword, "auto", TokenType::AutoKeyword);
lex_token_test!(lex_bool_keyword, "bool", TokenType::BoolKeyword);
lex_token_test!(lex_break_keyword, "break", TokenType::BreakKeyword);
lex_token_test!(lex_case_keyword, "case", TokenType::CaseKeyword);
lex_token_test!(lex_cast_keyword, "cast", TokenType::CastKeyword);
lex_token_test!(lex_catch_keyword, "catch", TokenType::CatchKeyword);
lex_token_test!(lex_class_keyword, "class", TokenType::ClassKeyword);
lex_token_test!(lex_const_keyword, "const", TokenType::ConstKeyword);
lex_token_test!(lex_continue_keyword, "continue", TokenType::ContinueKeyword);
lex_token_test!(lex_default_keyword, "default", TokenType::DefaultKeyword);
lex_token_test!(lex_do_keyword, "do", TokenType::DoKeyword);
lex_token_test!(lex_double_keyword, "double", TokenType::DoubleKeyword);
lex_token_test!(lex_else_keyword, "else", TokenType::ElseKeyword);
lex_token_test!(lex_enum_keyword, "enum", TokenType::EnumKeyword);
lex_token_test!(lex_explicit_keyword, "explicit", TokenType::ExplicitKeyword);
lex_token_test!(lex_external_keyword, "external", TokenType::ExternalKeyword);
lex_token_test!(lex_false_keyword, "false", TokenType::FalseKeyword);
lex_token_test!(lex_final_keyword, "final", TokenType::FinalKeyword);
lex_token_test!(lex_float_keyword, "float", TokenType::FloatKeyword);
lex_token_test!(lex_for_keyword, "for", TokenType::ForKeyword);
lex_token_test!(lex_from_keyword, "from", TokenType::FromKeyword);
lex_token_test!(lex_funcdef_keyword, "funcdef", TokenType::FuncDefKeyword);
lex_token_test!(lex_function_keyword, "function", TokenType::FunctionKeyword);
lex_token_test!(lex_get_keyword, "get", TokenType::GetKeyword);
lex_token_test!(lex_if_keyword, "if", TokenType::IfKeyword);
lex_token_test!(lex_import_keyword, "import", TokenType::ImportKeyword);
lex_token_test!(lex_in_keyword, "in", TokenType::InKeyword);
lex_token_test!(lex_inout_keyword, "inout", TokenType::InOutKeyword);
lex_token_test!(lex_int_keyword, "int", TokenType::IntKeyword);
lex_token_test!(
    lex_interface_keyword,
    "interface",
    TokenType::InterfaceKeyword
);
lex_token_test!(lex_int8_keyword, "int8", TokenType::Int8Keyword);
lex_token_test!(lex_int16_keyword, "int16", TokenType::Int16Keyword);
lex_token_test!(lex_int32_keyword, "int32", TokenType::Int32Keyword);
lex_token_test!(lex_int64_keyword, "int64", TokenType::Int64Keyword);
lex_token_test!(lex_is_keyword, "is", TokenType::IsKeyword);
lex_token_test!(lex_mixin_keyword, "mixin", TokenType::MixinKeyword);
lex_token_test!(
    lex_namespace_keyword,
    "namespace",
    TokenType::NamespaceKeyword
);
lex_token_test!(lex_not_keyword, "not", TokenType::NotKeyword);
lex_token_test!(lex_null_keyword, "null", TokenType::NullKeyword);
lex_token_test!(lex_or_keyword, "or", TokenType::OrKeyword);
lex_token_test!(lex_out_keyword, "out", TokenType::OutKeyword);
lex_token_test!(lex_override_keyword, "override", TokenType::OverrideKeyword);
lex_token_test!(lex_private_keyword, "private", TokenType::PrivateKeyword);
lex_token_test!(lex_property_keyword, "property", TokenType::PropertyKeyword);
lex_token_test!(
    lex_protected_keyword,
    "protected",
    TokenType::ProtectedKeyword
);

lex_token_test!(lex_return_keyword, "return", TokenType::ReturnKeyword);
lex_token_test!(lex_set_keyword, "set", TokenType::SetKeyword);
lex_token_test!(lex_shared_keyword, "shared", TokenType::SharedKeyword);
lex_token_test!(lex_super_keyword, "super", TokenType::SuperKeyword);
lex_token_test!(lex_switch_keyword, "switch", TokenType::SwitchKeyword);
lex_token_test!(lex_this_keyword, "this", TokenType::ThisKeyword);
lex_token_test!(lex_true_keyword, "true", TokenType::TrueKeyword);
lex_token_test!(lex_try_keyword, "try", TokenType::TryKeyword);
lex_token_test!(lex_typedef_keyword, "typedef", TokenType::TypeDefKeyword);
lex_token_test!(lex_uint_keyword, "uint", TokenType::UintKeyword);
lex_token_test!(lex_uint8_keyword, "uint8", TokenType::Uint8Keyword);
lex_token_test!(lex_uint16_keyword, "uint16", TokenType::Uint16Keyword);
lex_token_test!(lex_uint32_keyword, "uint32", TokenType::Uint32Keyword);

lex_token_test!(lex_void_keyword, "void", TokenType::VoidKeyword);
lex_token_test!(lex_while_keyword, "while", TokenType::WhileKeyword);
lex_token_test!(lex_xor_keyword, "xor", TokenType::XorKeyword);

lex_identifier_test!(lex_basic_identifier_foo, "foo");
lex_identifier_test!(lex_basic_identifier_foobar, "foobar");

lex_integer_test!(lex_zero, "0", 0);
lex_integer_test!(lex_one_two_three_four, "1234", 1234);
lex_integer_test!(lex_specific_one_two_three_four, "0d1234", 1234);
lex_integer_test!(lex_decimal_with_underline, "123_456", 123456);
lex_integer_test!(lex_specific_decimal_with_underline, "0D123_456", 123456);
lex_integer_test!(lex_hexadecimal_0f, "0X0F", 15);
lex_integer_test!(lex_hexadecimal_ff, "0xff", 255);
lex_integer_test!(lex_hexadecimal_ff_ff, "0xff_ff", 65535);
lex_integer_test!(lex_octal_112, "0o112", 74);
lex_integer_test!(lex_binary_1110, "0b1110", 14);
lex_integer_test!(lex_binary_01110, "0b01110", 14);

lex_float_test!(lex_zero_float, "0.0", 0.0);
lex_float_test!(lex_half, "0.5", 0.5);
lex_float_test!(lex_point_0_5, "0.05", 0.05);
lex_float_test!(lex_half_with_exponent, "0.5e10", 0.5e10);

lex_string_test!(lex_simple_string, "\"foo\"", "foo");
lex_string_test!(lex_simple_string_single_quote, "\'foo\'", "foo");
lex_string_test!(lex_string_with_escape, "\"fo\\\"o\"", "fo\"o");
lex_string_test!(lex_string_with_new_line, "\"fo\\no\"", "fo\no");
lex_string_test!(lex_heredoc_string, "\"\"\"foo\"\"\"", "foo");
lex_string_test!(lex_heredoc_string_with_quote, "\"\"\"fo\"o\"\"\"", "fo\"o");

#[test]
fn lex_two_identifier() {
    let tokens = lex("foo bar", &mut |_message, _span| {
        std::panic::panic_any(_message.stringify());
    });
    assert_eq!(tokens.len(), 4);
    assert_eq!(
        tokens[0].token_type,
        TokenType::Identifier("foo".to_string())
    );
    assert_eq!(tokens[0].span.start, 0);
    assert_eq!(tokens[0].span.end, 3);
    assert_eq!(tokens[1].token_type, TokenType::WhiteSpace);
    assert_eq!(
        tokens[2].token_type,
        TokenType::Identifier("bar".to_string())
    );
    assert_eq!(tokens[2].span.start, 4);
    assert_eq!(tokens[2].span.end, 7);

    assert_eq!(tokens[3].token_type, TokenType::EndOfFile);
}

#[test]
fn lex_multiple_tokens_with_not_is() {
    let tokens = lex("a !is b", &mut |_message, _span| {
        std::panic::panic_any(_message.stringify());
    });
    assert_eq!(tokens.len(), 6);
    assert_eq!(tokens[0].token_type, TokenType::Identifier("a".to_string()));
    assert_eq!(tokens[0].span.start, 0);
    assert_eq!(tokens[0].span.end, 1);
    assert_eq!(tokens[1].token_type, TokenType::WhiteSpace);
    assert_eq!(tokens[1].span.start, 1);
    assert_eq!(tokens[1].span.end, 2);
    assert_eq!(tokens[2].token_type, TokenType::NotIsKeyword);
    assert_eq!(tokens[2].span.start, 2);
    assert_eq!(tokens[2].span.end, 5);
    assert_eq!(tokens[3].token_type, TokenType::WhiteSpace);
    assert_eq!(tokens[3].span.start, 5);
    assert_eq!(tokens[3].span.end, 6);
    assert_eq!(tokens[4].token_type, TokenType::Identifier("b".to_string()));
    assert_eq!(tokens[4].span.start, 6);
    assert_eq!(tokens[4].span.end, 7);
    assert_eq!(tokens[5].token_type, TokenType::EndOfFile);
    assert_eq!(tokens[5].span.start, 7);
    assert_eq!(tokens[5].span.end, 7);
}

#[test]
fn lex_invalid_character_at_first_position() {
    let mut reached = false;
    lex("\x08", &mut |message, span| {
        reached = true;
        assert_matches!(message, Message::UnexpectedCharacter('\x08'));
        assert_eq!(span.start, 0);
        assert_eq!(span.end, 1);
    });
    assert!(reached);
}

#[test]
fn lex_invalid_character_at_other_position() {
    let mut reached = false;
    lex("  \x08", &mut |message, span| {
        reached = true;
        assert_matches!(message, Message::UnexpectedCharacter('\x08'));
        assert_eq!(span.start, 2);
        assert_eq!(span.end, 3);
    });
    assert!(reached);
}

#[test]
fn lex_unclosed_string_literal() {
    let mut reached = false;
    lex("\"    ", &mut |message, span| {
        reached = true;
        assert_matches!(message, Message::UnclosedStringLiteral);
        assert_eq!(span.start, 5);
        assert_eq!(span.end, 6);
    });
    assert!(reached);
}