diff --git a/src/Parser/Lexer/LexTokenKind.hpp b/src/Parser/Lexer/LexTokenKind.hpp index c6eb8fc..983e415 100644 --- a/src/Parser/Lexer/LexTokenKind.hpp +++ b/src/Parser/Lexer/LexTokenKind.hpp @@ -9,6 +9,7 @@ namespace MalachScript::Parser { Unknown, EndOfFile, Whitespace, + Comment, // Symbols StarSymbol, @@ -265,6 +266,7 @@ namespace MalachScript::Parser { case LexTokenKind::IntegerLiteral: return "integer"; case LexTokenKind::StringLiteral: return "string"; case LexTokenKind::Identifier: return "identifier"; + case LexTokenKind::Comment: return "Comment"; } return "FAIL"; } diff --git a/src/Parser/Lexer/Lexer.cpp b/src/Parser/Lexer/Lexer.cpp index 3e430fc..17ded3c 100644 --- a/src/Parser/Lexer/Lexer.cpp +++ b/src/Parser/Lexer/Lexer.cpp @@ -57,6 +57,25 @@ namespace MalachScript::Parser { // /= return Create>( ScriptTextSpan(start, start + 2, _scriptName)); + } else if (Peek() == '/') { + Progress(); + while (true) { + auto next = Consume(); + if (next == '\n' || next == '\0') { + return Create>( + ScriptTextSpan(start, _position, _scriptName)); + } + } + } else if (Peek() == '*') { + Progress(); + while (true) { + auto next = Consume(); + if ((next == '*' && Peek() == '/') || next == '\0') { + Progress(); + return Create>( + ScriptTextSpan(start, _position, _scriptName)); + } + } } // / return Create>(ScriptTextSpan(start, start + 1, _scriptName)); diff --git a/src/Parser/Parser.cpp b/src/Parser/Parser.cpp index d056e75..1f44fb1 100644 --- a/src/Parser/Parser.cpp +++ b/src/Parser/Parser.cpp @@ -5,7 +5,7 @@ #define PROGRESS_TOKEN(token) \ token = (token)->GetNext().get(); \ - while ((token)->GetKind() == LexTokenKind::Whitespace) { \ + while ((token)->GetKind() == LexTokenKind::Whitespace || (token)->GetKind() == LexTokenKind::Comment) { \ (token) = (token)->GetNext().get(); \ } @@ -57,7 +57,7 @@ namespace MalachScript::Parser { size_t currentAmount = 0; const auto* current = currentToken; while (true) { - while (current->GetKind() == LexTokenKind::Whitespace) { + while (current->GetKind() == LexTokenKind::Whitespace || current->GetKind() == LexTokenKind::Comment) { current = current->GetNext().get(); } if (current->GetKind() == LexTokenKind::EndOfFile) { diff --git a/tests/ParserTests/ClassTests.cpp b/tests/ParserTests/ClassTests.cpp index 2fff150..6053fb3 100644 --- a/tests/ParserTests/ClassTests.cpp +++ b/tests/ParserTests/ClassTests.cpp @@ -13,7 +13,7 @@ using namespace MalachScript; vec[i]->SetNext(vec[i + 1]); \ } \ Diagnostics::Logger diags; \ - auto* script = Parser::Parser::Parse(vec.front(), "scriptname", &diags); \ + auto* script = Parser::Parser::Parse(vec.front(), &diags); \ REQUIRE(diags.GetMessages().empty()); \ asserts; \ delete vec[0]; \ @@ -21,6 +21,7 @@ using namespace MalachScript; } #define PARSER_TEST_TOKENS(...) __VA_ARGS__ +#define TextSpan(a, b) ScriptTextSpan(a, b, "") PARSER_TEST("Parse basic class without body", PARSER_TEST_TOKENS(new Parser::LexTokenImpl(TextSpan(0, 0)), diff --git a/tests/ParserTests/FunctionTests.cpp b/tests/ParserTests/FunctionTests.cpp index 57521fc..bb1e589 100644 --- a/tests/ParserTests/FunctionTests.cpp +++ b/tests/ParserTests/FunctionTests.cpp @@ -13,7 +13,7 @@ using namespace MalachScript; vec[i]->SetNext(vec[i + 1]); \ } \ Diagnostics::Logger diags; \ - auto* script = Parser::Parser::Parse(vec.front(), "scriptname", &diags); \ + auto* script = Parser::Parser::Parse(vec.front(), &diags); \ REQUIRE(diags.GetMessages().empty()); \ asserts; \ delete vec[0]; \ @@ -21,6 +21,7 @@ using namespace MalachScript; } #define PARSER_TEST_TOKENS(...) __VA_ARGS__ +#define TextSpan(a, b) ScriptTextSpan(a, b, "") PARSER_TEST("Parse ``void foobar();``", PARSER_TEST_TOKENS(new Parser::LexTokenImpl(TextSpan(0, 0)), diff --git a/tests/ParserTests/ParserIntegrationTests.cpp b/tests/ParserTests/ParserIntegrationTests.cpp index 63c09a5..22eb6f8 100644 --- a/tests/ParserTests/ParserIntegrationTests.cpp +++ b/tests/ParserTests/ParserIntegrationTests.cpp @@ -9,7 +9,7 @@ using namespace MalachScript; Diagnostics::Logger diags; \ auto lexer = Parser::Lexer(name, scriptText, &diags); \ auto token = lexer.Lex(); \ - auto script = Parser::Parser::Parse(token, name, &diags); \ + auto script = Parser::Parser::Parse(token, &diags); \ asserts; \ delete script; \ } diff --git a/tests/ParserTests/VirtPropTests.cpp b/tests/ParserTests/VirtPropTests.cpp index 4c8d4c4..77008cb 100644 --- a/tests/ParserTests/VirtPropTests.cpp +++ b/tests/ParserTests/VirtPropTests.cpp @@ -13,7 +13,7 @@ using namespace MalachScript; vec[i]->SetNext(vec[i + 1]); \ } \ Diagnostics::Logger diags; \ - auto* script = Parser::Parser::Parse(vec.front(), "scriptname", &diags); \ + auto* script = Parser::Parser::Parse(vec.front(), &diags); \ REQUIRE(diags.GetMessages().empty()); \ asserts; \ delete vec[0]; \ @@ -21,113 +21,111 @@ using namespace MalachScript; } #define PARSER_TEST_TOKENS(...) __VA_ARGS__ +#define TextSpan(a, b) ScriptTextSpan(a, b, "") -PARSER_TEST( - "Parse class foobar { bool foo { get; set; } }", - PARSER_TEST_TOKENS(new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::IdentifierToken(TextSpan(0, 0), "foobar"), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::IdentifierToken(TextSpan(0, 0), "bool"), - new Parser::IdentifierToken(TextSpan(0, 0), "foo"), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0))), - { - REQUIRE(script->GetStatements().size() == 1); - auto firstStatement = script->GetStatements()[0].get(); - REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); - auto firstClassStatement = - dynamic_cast(firstStatement)->GetBody()[0].get(); - REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); - auto virtPropStatement = - dynamic_cast(firstClassStatement); - REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); - REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); - REQUIRE(virtPropStatement->HasGet()); - REQUIRE(virtPropStatement->HasSet()); - REQUIRE_FALSE(virtPropStatement->IsGetConst()); - REQUIRE_FALSE(virtPropStatement->IsSetConst()); - REQUIRE(virtPropStatement->GetGetStatement() == nullptr); - REQUIRE(virtPropStatement->GetSetStatement() == nullptr); - }) +PARSER_TEST("Parse class foobar { bool foo { get; set; } }", + PARSER_TEST_TOKENS( + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::IdentifierToken(TextSpan(0, 0), "foobar"), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::IdentifierToken(TextSpan(0, 0), "bool"), new Parser::IdentifierToken(TextSpan(0, 0), "foo"), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0))), + { + REQUIRE(script->GetStatements().size() == 1); + auto firstStatement = script->GetStatements()[0].get(); + REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); + auto firstClassStatement = + dynamic_cast(firstStatement)->GetBody()[0].get(); + REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); + auto virtPropStatement = + dynamic_cast(firstClassStatement); + REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); + REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); + REQUIRE(virtPropStatement->HasGet()); + REQUIRE(virtPropStatement->HasSet()); + REQUIRE_FALSE(virtPropStatement->IsGetConst()); + REQUIRE_FALSE(virtPropStatement->IsSetConst()); + REQUIRE(virtPropStatement->GetGetStatement() == nullptr); + REQUIRE(virtPropStatement->GetSetStatement() == nullptr); + }) -PARSER_TEST( - "Parse class foobar { bool foo { get const; set const; } }", - PARSER_TEST_TOKENS(new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::IdentifierToken(TextSpan(0, 0), "foobar"), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::IdentifierToken(TextSpan(0, 0), "bool"), - new Parser::IdentifierToken(TextSpan(0, 0), "foo"), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0))), - { - REQUIRE(script->GetStatements().size() == 1); - auto firstStatement = script->GetStatements()[0].get(); - REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); - auto firstClassStatement = - dynamic_cast(firstStatement)->GetBody()[0].get(); - REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); - auto virtPropStatement = - dynamic_cast(firstClassStatement); - REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); - REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); - REQUIRE(virtPropStatement->HasGet()); - REQUIRE(virtPropStatement->HasSet()); - REQUIRE(virtPropStatement->IsGetConst()); - REQUIRE(virtPropStatement->IsSetConst()); - REQUIRE(virtPropStatement->GetGetStatement() == nullptr); - REQUIRE(virtPropStatement->GetSetStatement() == nullptr); - }) +PARSER_TEST("Parse class foobar { bool foo { get const; set const; } }", + PARSER_TEST_TOKENS( + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::IdentifierToken(TextSpan(0, 0), "foobar"), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::IdentifierToken(TextSpan(0, 0), "bool"), new Parser::IdentifierToken(TextSpan(0, 0), "foo"), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0))), + { + REQUIRE(script->GetStatements().size() == 1); + auto firstStatement = script->GetStatements()[0].get(); + REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); + auto firstClassStatement = + dynamic_cast(firstStatement)->GetBody()[0].get(); + REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); + auto virtPropStatement = + dynamic_cast(firstClassStatement); + REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); + REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); + REQUIRE(virtPropStatement->HasGet()); + REQUIRE(virtPropStatement->HasSet()); + REQUIRE(virtPropStatement->IsGetConst()); + REQUIRE(virtPropStatement->IsSetConst()); + REQUIRE(virtPropStatement->GetGetStatement() == nullptr); + REQUIRE(virtPropStatement->GetSetStatement() == nullptr); + }) -PARSER_TEST( - "Parse class foobar { bool foo { get const override; set const override; } }", - PARSER_TEST_TOKENS(new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::IdentifierToken(TextSpan(0, 0), "foobar"), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::IdentifierToken(TextSpan(0, 0), "bool"), - new Parser::IdentifierToken(TextSpan(0, 0), "foo"), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0))), - { - REQUIRE(script->GetStatements().size() == 1); - auto firstStatement = script->GetStatements()[0].get(); - REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); - auto firstClassStatement = - dynamic_cast(firstStatement)->GetBody()[0].get(); - REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); - auto virtPropStatement = - dynamic_cast(firstClassStatement); - REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); - REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); - REQUIRE(virtPropStatement->HasGet()); - REQUIRE(virtPropStatement->HasSet()); - REQUIRE(virtPropStatement->IsGetConst()); - REQUIRE(virtPropStatement->IsSetConst()); - REQUIRE(FuncAttrHelpers::Contains(virtPropStatement->GetGetFuncAttr(), FuncAttr::Override)); - REQUIRE(FuncAttrHelpers::Contains(virtPropStatement->GetSetFuncAttr(), FuncAttr::Override)); - REQUIRE(virtPropStatement->GetGetStatement() == nullptr); - REQUIRE(virtPropStatement->GetSetStatement() == nullptr); - }) +PARSER_TEST("Parse class foobar { bool foo { get const override; set const override; } }", + PARSER_TEST_TOKENS( + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::IdentifierToken(TextSpan(0, 0), "foobar"), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::IdentifierToken(TextSpan(0, 0), "bool"), new Parser::IdentifierToken(TextSpan(0, 0), "foo"), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0))), + { + REQUIRE(script->GetStatements().size() == 1); + auto firstStatement = script->GetStatements()[0].get(); + REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); + auto firstClassStatement = + dynamic_cast(firstStatement)->GetBody()[0].get(); + REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); + auto virtPropStatement = + dynamic_cast(firstClassStatement); + REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); + REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); + REQUIRE(virtPropStatement->HasGet()); + REQUIRE(virtPropStatement->HasSet()); + REQUIRE(virtPropStatement->IsGetConst()); + REQUIRE(virtPropStatement->IsSetConst()); + REQUIRE(FuncAttrHelpers::Contains(virtPropStatement->GetGetFuncAttr(), FuncAttr::Override)); + REQUIRE(FuncAttrHelpers::Contains(virtPropStatement->GetSetFuncAttr(), FuncAttr::Override)); + REQUIRE(virtPropStatement->GetGetStatement() == nullptr); + REQUIRE(virtPropStatement->GetSetStatement() == nullptr); + }) /// class foobar { // int i; @@ -143,9 +141,8 @@ PARSER_TEST( // } //} -PARSER_TEST( - "Virtprops with bodies", - PARSER_TEST_TOKENS( +PARSER_TEST("Virtprops with bodies", + PARSER_TEST_TOKENS( new Parser::LexTokenImpl(TextSpan(0, 0)), new Parser::IdentifierToken(TextSpan(0, 0), "foobar"), new Parser::LexTokenImpl(TextSpan(0, 0)), @@ -164,8 +161,8 @@ PARSER_TEST( new Parser::LexTokenImpl(TextSpan(0, 0)), new Parser::LexTokenImpl(TextSpan(0, 0)), new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), - new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), + new Parser::LexTokenImpl(TextSpan(0, 0)), new Parser::LexTokenImpl(TextSpan(0, 0)), new Parser::LexTokenImpl(TextSpan(0, 0)), @@ -186,21 +183,21 @@ PARSER_TEST( new Parser::LexTokenImpl(TextSpan(0, 0)), new Parser::LexTokenImpl(TextSpan(0, 0)), new Parser::LexTokenImpl(TextSpan(0, 0))), - { - REQUIRE(script->GetStatements().size() == 1); - auto firstStatement = script->GetStatements()[0].get(); - REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); - auto firstClassStatement = - dynamic_cast(firstStatement)->GetBody()[1].get(); - REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); - auto virtPropStatement = - dynamic_cast(firstClassStatement); - REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); - REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); - REQUIRE(virtPropStatement->HasGet()); - REQUIRE(virtPropStatement->HasSet()); - REQUIRE_FALSE(virtPropStatement->IsGetConst()); - REQUIRE_FALSE(virtPropStatement->IsSetConst()); - REQUIRE(virtPropStatement->GetGetStatement() != nullptr); - REQUIRE(virtPropStatement->GetSetStatement() != nullptr); - }) \ No newline at end of file + { + REQUIRE(script->GetStatements().size() == 1); + auto firstStatement = script->GetStatements()[0].get(); + REQUIRE(firstStatement->GetKind() == Parser::ParsedStatementKind::Class); + auto firstClassStatement = + dynamic_cast(firstStatement)->GetBody()[1].get(); + REQUIRE(firstClassStatement->GetKind() == Parser::ParsedStatementKind::VirtProp); + auto virtPropStatement = + dynamic_cast(firstClassStatement); + REQUIRE(virtPropStatement->GetAccess() == MalachScript::AccessModifier::Public); + REQUIRE(virtPropStatement->GetIdentifier().GetString() == "foo"); + REQUIRE(virtPropStatement->HasGet()); + REQUIRE(virtPropStatement->HasSet()); + REQUIRE_FALSE(virtPropStatement->IsGetConst()); + REQUIRE_FALSE(virtPropStatement->IsSetConst()); + REQUIRE(virtPropStatement->GetGetStatement() != nullptr); + REQUIRE(virtPropStatement->GetSetStatement() != nullptr); + }) \ No newline at end of file