Deukhoofd/PorygonLang
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make Lexer use constant Tokens
Some checks failed
continuous-integration/drone/push Build is failing
Details

Browse Source
This commit is contained in:
Deukhoofd
2019-06-13 18:49:38 +02:00
parent 5910cbbfa9
commit 3e00f750ef
10 changed files with 162 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
tests/parser/LexerTests.cpp
View File

@@ -58,7 +58,7 @@ TEST_CASE( "Lex Equality Token", "[lexer]" ) {
Lexer lexer = Lexer("==", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::EqualityToken);
}
@@ -92,10 +92,10 @@ TEST_CASE( "Lex Longer Integers", "[lexer]" ) {
Lexer lexer = Lexer(std::to_string(integer), nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::Integer);
auto* integerToken = (IntegerToken *)firstToken;
CHECK(integerToken -> Value == integer);
CHECK(integerToken -> GetValue() == integer);
}
}
@@ -105,10 +105,10 @@ TEST_CASE( "Lex Floats", "[lexer]" ) {
Lexer lexer = Lexer(std::to_string(f), nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::Float);
auto* floatToken = (FloatToken *)firstToken;
CHECK(floatToken -> Value == Approx(f));
CHECK(floatToken -> GetValue() == Approx(f));
}
}
@@ -116,133 +116,133 @@ TEST_CASE( "Lex And Keyword", "[lexer]" ) {
Lexer lexer = Lexer("and", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::AndKeyword);
}
TEST_CASE( "Lex Break Keyword", "[lexer]" ) {
Lexer lexer = Lexer("break", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::BreakKeyword);
}
TEST_CASE( "Lex Do Keyword", "[lexer]" ) {
Lexer lexer = Lexer("do", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::DoKeyword);
}
TEST_CASE( "Lex else Keyword", "[lexer]" ) {
Lexer lexer = Lexer("else", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ElseKeyword);
}
TEST_CASE( "Lex else if Keyword", "[lexer]" ) {
Lexer lexer = Lexer("elseif", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ElseIfKeyword);
}
TEST_CASE( "Lex end Keyword", "[lexer]" ) {
Lexer lexer = Lexer("end", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::EndKeyword);
}
TEST_CASE( "Lex false Keyword", "[lexer]" ) {
Lexer lexer = Lexer("false", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::FalseKeyword);
}
TEST_CASE( "Lex for Keyword", "[lexer]" ) {
Lexer lexer = Lexer("for", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ForKeyword);
}
TEST_CASE( "Lex function Keyword", "[lexer]" ) {
Lexer lexer = Lexer("function", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::FunctionKeyword);
}
TEST_CASE( "Lex if Keyword", "[lexer]" ) {
Lexer lexer = Lexer("if", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::IfKeyword);
}
TEST_CASE( "Lex in Keyword", "[lexer]" ) {
Lexer lexer = Lexer("in", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::InKeyword);
}
TEST_CASE( "Lex local Keyword", "[lexer]" ) {
Lexer lexer = Lexer("local", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::LocalKeyword);
}
TEST_CASE( "Lex nil Keyword", "[lexer]" ) {
Lexer lexer = Lexer("nil", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::NilKeyword);
}
TEST_CASE( "Lex not Keyword", "[lexer]" ) {
Lexer lexer = Lexer("not", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::NotKeyword);
}
TEST_CASE( "Lex or Keyword", "[lexer]" ) {
Lexer lexer = Lexer("or", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::OrKeyword);
}
TEST_CASE( "Lex return Keyword", "[lexer]" ) {
Lexer lexer = Lexer("return", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ReturnKeyword);
}
TEST_CASE( "Lex then Keyword", "[lexer]" ) {
Lexer lexer = Lexer("then", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ThenKeyword);
}
TEST_CASE( "Lex true Keyword", "[lexer]" ) {
Lexer lexer = Lexer("true", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::TrueKeyword);
}
TEST_CASE( "Lex while Keyword", "[lexer]" ) {
Lexer lexer = Lexer("while", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::WhileKeyword);
}
@@ -250,9 +250,9 @@ TEST_CASE( "Lex identifier", "[lexer]" ) {
Lexer lexer = Lexer("foo", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::Identifier);
REQUIRE(((IdentifierToken*)firstToken) -> Value == "foo");
REQUIRE(((IdentifierToken*)firstToken) -> GetValue() == HashedString("foo"));
}
TEST_CASE( "Lex Start Position", "[lexer]" ) {
@@ -281,36 +281,36 @@ TEST_CASE("Lex Double Quote String", "[lexer]") {
Lexer lexer = Lexer("\"foo bar\"", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> Value == "foo bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar");
}
TEST_CASE("Lex Single Quote String", "[lexer]") {
Lexer lexer = Lexer("'foo bar'", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> Value == "foo bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar");
}
TEST_CASE("Lex Double Quote String, Escape Quote", "[lexer]") {
Lexer lexer = Lexer("'foo\\\"bar'", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> Value == "foo\"bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\"bar");
}
TEST_CASE("Lex String with newline", "[lexer]") {
Lexer lexer = Lexer("'foo\\nbar'", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
IToken* firstToken = tokens[0];
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> Value == "foo\nbar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\nbar");
}

40
tests/parser/ParserTests.cpp
View File

@@ -5,7 +5,7 @@
#include "../../src/Parser/Parser.hpp"
TEST_CASE( "Parse single true keyword", "[parser]" ) {
vector<IToken*> v {new SimpleToken(TokenKind::TrueKeyword,0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
vector<const IToken*> v {new SimpleToken(TokenKind::TrueKeyword,0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
Parser parser = Parser(v, nullptr);
auto parsedStatements = parser.Parse() -> GetStatements();
REQUIRE(parsedStatements->size() == 1);
@@ -18,7 +18,7 @@ TEST_CASE( "Parse single true keyword", "[parser]" ) {
}
TEST_CASE( "Parse single false keyword", "[parser]" ) {
vector<IToken*> v {new SimpleToken(TokenKind::FalseKeyword,0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
vector<const IToken*> v {new SimpleToken(TokenKind::FalseKeyword,0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
Parser parser = Parser(v, nullptr);
auto parsedStatements = parser.Parse() -> GetStatements();
REQUIRE(parsedStatements->size() == 1);
@@ -31,7 +31,7 @@ TEST_CASE( "Parse single false keyword", "[parser]" ) {
}
TEST_CASE( "Parse simple addition", "[parser]" ) {
vector<IToken*> v {
vector<const IToken*> v {
new IntegerToken(5, 0, 0),
new SimpleToken(TokenKind::PlusToken,0,0),
new IntegerToken(10, 0, 0),
@@ -55,7 +55,7 @@ TEST_CASE( "Parse simple addition", "[parser]" ) {
}
TEST_CASE( "Parse simple negation", "[parser]" ) {
vector<IToken*> v {
vector<const IToken*> v {
new SimpleToken(TokenKind::MinusToken,0,0),
new IntegerToken(10, 0, 0),
new SimpleToken(TokenKind::EndOfFile,0,0)
@@ -75,7 +75,7 @@ TEST_CASE( "Parse simple negation", "[parser]" ) {
}
TEST_CASE( "Parse logical negation", "[parser]" ) {
vector<IToken*> v {
vector<const IToken*> v {
new SimpleToken(TokenKind::NotKeyword,0,0),
new SimpleToken(TokenKind::FalseKeyword,0,0),
new SimpleToken(TokenKind::EndOfFile,0,0)
@@ -95,7 +95,7 @@ TEST_CASE( "Parse logical negation", "[parser]" ) {
}
TEST_CASE( "Are parenthesized expressions valid", "[parser]" ) {
vector<IToken*> v {
vector<const IToken*> v {
new IntegerToken(5, 0, 0),
new SimpleToken(TokenKind::PlusToken,0,0),
new IntegerToken(10, 0, 0),
@@ -124,7 +124,7 @@ TEST_CASE( "Are parenthesized expressions valid", "[parser]" ) {
}
TEST_CASE( "Assert binary precedence", "[parser]" ) {
vector<IToken*> v {
vector<const IToken*> v {
new SimpleToken(TokenKind::OpenParenthesis,0,0),
new IntegerToken(10, 0, 0),
new SimpleToken(TokenKind::CloseParenthesis,0,0),
@@ -142,7 +142,7 @@ TEST_CASE( "Assert binary precedence", "[parser]" ) {
}
TEST_CASE( "Parse String Tokens", "[parser]" ) {
vector<IToken*> v {new StringToken("foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
vector<const IToken*> v {new StringToken("foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
Parser parser = Parser(v, nullptr);
auto parsedStatements = parser.Parse() -> GetStatements();
REQUIRE(parsedStatements->size() == 1);
@@ -155,8 +155,8 @@ TEST_CASE( "Parse String Tokens", "[parser]" ) {
}
TEST_CASE( "Parse Global Assignment", "[parser]" ) {
vector<IToken*> v {
new IdentifierToken("foo",0,0),
vector<const IToken*> v {
new IdentifierToken(HashedString("foo"),0,0),
new SimpleToken(TokenKind::AssignmentToken,0,0),
new SimpleToken(TokenKind::TrueKeyword,0,0),
new SimpleToken(TokenKind::EndOfFile,0,0)
@@ -173,9 +173,9 @@ TEST_CASE( "Parse Global Assignment", "[parser]" ) {
}
TEST_CASE( "Parse local Assignment", "[parser]" ) {
vector<IToken*> v {
vector<const IToken*> v {
new SimpleToken(TokenKind::LocalKeyword,0,0),
new IdentifierToken("foo",0,0),
new IdentifierToken(HashedString("foo"),0,0),
new SimpleToken(TokenKind::AssignmentToken,0,0),
new SimpleToken(TokenKind::TrueKeyword,0,0),
new SimpleToken(TokenKind::EndOfFile,0,0)
@@ -192,19 +192,19 @@ TEST_CASE( "Parse local Assignment", "[parser]" ) {
}
TEST_CASE( "Parse function declaration", "[parser]" ){
vector<IToken*> v {
vector<const IToken*> v {
new SimpleToken(TokenKind::FunctionKeyword,0,0),
new IdentifierToken("foo",0,0),
new IdentifierToken(HashedString("foo"),0,0),
new SimpleToken(TokenKind::OpenParenthesis,0,0),
new IdentifierToken("number",0,0),
new IdentifierToken("bar",0,0),
new IdentifierToken(HashedString("number"),0,0),
new IdentifierToken(HashedString("bar"),0,0),
new SimpleToken(TokenKind::CommaToken,0,0),
new IdentifierToken("number",0,0),
new IdentifierToken("par",0,0),
new IdentifierToken(HashedString("number"),0,0),
new IdentifierToken(HashedString("par"),0,0),
new SimpleToken(TokenKind::CloseParenthesis,0,0),
new IdentifierToken("bar",0,0),
new IdentifierToken(HashedString("bar"),0,0),
new SimpleToken(TokenKind::PlusToken,0,0),
new IdentifierToken("par",0,0),
new IdentifierToken(HashedString("par"),0,0),
new SimpleToken(TokenKind::EndKeyword,0,0),
new SimpleToken(TokenKind::EndOfFile,0,0),
};