Deukhoofd/PorygonLang
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move Lexer to u16string handling, for unicode support
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Deukhoofd
2019-06-15 17:20:27 +02:00
parent f73bd2003c
commit 3dc67ec8a0
21 changed files with 189 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/integration/ConditionalTests.cpp
View File

@@ -5,10 +5,10 @@
TEST_CASE( "Basic conditional", "[integration]" ) {
Script* script = Script::Create("if true then foo = true end");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;
@@ -17,10 +17,10 @@ TEST_CASE( "Basic conditional", "[integration]" ) {
TEST_CASE( "If then, else", "[integration]" ) {
Script* script = Script::Create("if false then foo = false else foo = true end");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;
@@ -29,10 +29,10 @@ TEST_CASE( "If then, else", "[integration]" ) {
TEST_CASE( "If then, else if", "[integration]" ) {
Script* script = Script::Create("if false then foo = false elseif true then foo = true end");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;

30
tests/integration/Functions.cpp
View File

@@ -6,7 +6,7 @@ TEST_CASE( "Define script function", "[integration]" ) {
Script* script = Script::Create("function add(number a, number b) a + b end");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("add");
auto variable = script->GetVariable(u"add");
REQUIRE(variable != nullptr);
REQUIRE(variable->GetTypeClass() == TypeClass::Function);
delete script;
@@ -16,10 +16,10 @@ TEST_CASE( "Define script function and call", "[integration]" ) {
Script* script = Script::Create("function add(number a, number b) result = a + b end add(1, 2)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("add");
auto variable = script->GetVariable(u"add");
REQUIRE(variable != nullptr);
REQUIRE(variable->GetTypeClass() == TypeClass::Function);
auto result = script->GetVariable("result");
auto result = script->GetVariable(u"result");
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 3);
delete script;
@@ -29,10 +29,10 @@ TEST_CASE( "Define script function and call multiple times", "[integration]" ) {
Script* script = Script::Create("result = 0 function add(number a) result = result + a end add(1) add(4)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("add");
auto variable = script->GetVariable(u"add");
REQUIRE(variable != nullptr);
REQUIRE(variable->GetTypeClass() == TypeClass::Function);
auto result = script->GetVariable("result");
auto result = script->GetVariable(u"result");
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 5);
delete script;
@@ -43,15 +43,15 @@ TEST_CASE( "Define script function and call from extern", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
REQUIRE(script->HasFunction("add"));
REQUIRE(script->HasFunction(u"add"));
auto toAddVal = new IntegerEvalValue(5);
script->CallFunction("add", {toAddVal});
script->CallFunction(u"add", {toAddVal});
delete toAddVal;
toAddVal = new IntegerEvalValue(6);
script->CallFunction("add", {toAddVal});
script->CallFunction(u"add", {toAddVal});
delete toAddVal;
auto result = script->GetVariable("result");
auto result = script->GetVariable(u"result");
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 11);
delete script;
@@ -67,17 +67,17 @@ TEST_CASE( "Define script function and return", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
REQUIRE(script->HasFunction("add"));
REQUIRE(script->HasFunction(u"add"));
auto toAddVal = new IntegerEvalValue(5);
auto toAddVal2 = new IntegerEvalValue(6);
auto result = script->CallFunction("add", {toAddVal, toAddVal2});
auto result = script->CallFunction(u"add", {toAddVal, toAddVal2});
delete toAddVal;
delete toAddVal2;
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 11);
auto variable = script->GetVariable("val");
auto variable = script->GetVariable(u"val");
REQUIRE(variable->GetTypeClass() == TypeClass::Number);
REQUIRE(variable->EvaluateInteger() == 0);
@@ -97,9 +97,9 @@ end
)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
script->CallFunction("add", {});
script->CallFunction(u"add", {});
auto variable = script->GetVariable("val");
auto variable = script->GetVariable(u"val");
REQUIRE(variable->GetTypeClass() == TypeClass::Number);
REQUIRE(variable->EvaluateInteger() == 5);
@@ -127,7 +127,7 @@ test()
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("result");
auto variable = script->GetVariable(u"result");
REQUIRE(variable->GetTypeClass() == TypeClass::Number);
REQUIRE(variable->EvaluateInteger() == 50);

2
tests/integration/IndexTests.cpp
View File

@@ -7,7 +7,7 @@ TEST_CASE( "String indexing", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "b");
REQUIRE(*lastValue->EvaluateString() == u"b");
delete script;
}

6
tests/integration/StringOperationsTests.cpp
View File

@@ -9,7 +9,7 @@ TEST_CASE( "Simple String", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "foo bar");
REQUIRE(*lastValue->EvaluateString() == u"foo bar");
delete script;
}
@@ -18,7 +18,7 @@ TEST_CASE( "String Concat", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "foobar");
REQUIRE(*lastValue->EvaluateString() == u"foobar");
delete script;
}
@@ -27,7 +27,7 @@ TEST_CASE( "String Concat 2", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "foobar");
REQUIRE(*lastValue->EvaluateString() == u"foobar");
delete script;
}

12
tests/integration/Tables.cpp
View File

@@ -7,7 +7,7 @@ TEST_CASE( "Create empty table", "[integration]" ) {
Script* script = Script::Create("table = {}");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
delete script;
}
@@ -16,7 +16,7 @@ TEST_CASE( "Create simple integer table", "[integration]" ) {
Script* script = Script::Create("table = {100, 50, 20, 5, -100, 50+50}");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
delete script;
}
@@ -25,7 +25,7 @@ TEST_CASE( "Create simple string table", "[integration]" ) {
Script* script = Script::Create("table = {'bla', 'test', 'foo', 'bar'}");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
delete script;
}
@@ -39,7 +39,7 @@ return table[3]
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->Evaluate();
REQUIRE(variable != nullptr);
REQUIRE(*variable->EvaluateString() == "foo");
REQUIRE(*variable->EvaluateString() == u"foo");
delete script;
}
@@ -53,10 +53,10 @@ table = {
)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
auto table = (TableEvalValue*)variable;
CHECK(*table->IndexValue("foo")->EvaluateString() == "test");
CHECK(*table->IndexValue("foo")->EvaluateString() == u"test");
CHECK(table->IndexValue("bar")->EvaluateInteger() == 100);
delete script;
}

4
tests/integration/UserData.cpp
View File

@@ -38,7 +38,7 @@ end
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto parameter = new UserDataValue(HashedString::ConstHash("testObject"), new UserDataTestObject());
auto variable = script->CallFunction("testFunc", {parameter});
auto variable = script->CallFunction(u"testFunc", {parameter});
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateInteger() == 10);
delete script;
@@ -55,7 +55,7 @@ end
script->Evaluate();
auto obj = new UserDataTestObject();
auto parameter = new UserDataValue(HashedString::ConstHash("testObject"), obj);
script->CallFunction("testFunc", {parameter});
script->CallFunction(u"testFunc", {parameter});
delete script;
REQUIRE(obj->foo == 5000);
delete obj;

16
tests/integration/Variables.cpp
View File

@@ -5,10 +5,10 @@
TEST_CASE( "Create script variable", "[integration]" ) {
Script* script = Script::Create("foo = true");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;
@@ -17,7 +17,7 @@ TEST_CASE( "Create script variable", "[integration]" ) {
TEST_CASE( "Create local variable", "[integration]" ) {
Script* script = Script::Create("local foo = true");
REQUIRE(!script->Diagnostics -> HasErrors());
REQUIRE_FALSE(script->HasVariable("foo"));
REQUIRE_FALSE(script->HasVariable(u"foo"));
script->Evaluate();
delete script;
}
@@ -27,7 +27,7 @@ TEST_CASE( "Create script variable and use", "[integration]" ) {
"bar = not foo");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("bar");
auto variable = script->GetVariable(u"bar");
REQUIRE(variable != nullptr);
CHECK(variable->EvaluateBool());
delete script;
@@ -38,7 +38,7 @@ TEST_CASE( "Create local variable and use", "[integration]" ) {
"bar = not foo");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("bar");
auto variable = script->GetVariable(u"bar");
REQUIRE(variable != nullptr);
CHECK(variable->EvaluateBool());
delete script;
@@ -55,9 +55,9 @@ end
)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
script -> CallFunction("bar", {});
script -> CallFunction("bar", {});
auto variable = script->GetVariable("result");
script -> CallFunction(u"bar", {});
script -> CallFunction(u"bar", {});
auto variable = script->GetVariable(u"result");
REQUIRE(variable != nullptr);
CHECK(variable->EvaluateInteger() == 2);
delete script;

108
tests/parser/LexerTests.cpp
View File

@@ -1,14 +1,16 @@
#ifdef TESTS_BUILD
#include <catch.hpp>
#include <codecvt>
#include <locale>
#include "../../src/Parser/Lexer.hpp"
TEST_CASE( "When at end of script return terminator", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.Peek() == '\0');
}
TEST_CASE( "Peek doesn't advance", "[lexer]" ) {
auto script = new string("5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
auto script = new u16string(u"5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
Lexer lexer = Lexer(*script, nullptr);
REQUIRE(lexer.Peek() == '5');
REQUIRE(lexer.Peek() == '5');
@@ -16,7 +18,7 @@ TEST_CASE( "Peek doesn't advance", "[lexer]" ) {
}
TEST_CASE( "Next does advance", "[lexer]" ) {
auto script = new string("5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
auto script = new u16string(u"5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
Lexer lexer = Lexer(*script, nullptr);
REQUIRE(lexer.Next() == '5');
REQUIRE(lexer.Next() == ' ');
@@ -27,37 +29,37 @@ TEST_CASE( "Next does advance", "[lexer]" ) {
}
TEST_CASE( "Lex Null Terminator as EOF", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('\0') -> GetKind() == TokenKind::EndOfFile);
}
TEST_CASE( "Lex Plus Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('+') -> GetKind() == TokenKind::PlusToken);
}
TEST_CASE( "Lex Minus Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('-') -> GetKind() == TokenKind::MinusToken);
}
TEST_CASE( "Lex Slash Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('/') -> GetKind() == TokenKind::SlashToken);
}
TEST_CASE( "Lex Star Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('*') -> GetKind() == TokenKind::StarToken);
}
TEST_CASE( "Lex Assignment Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('=') -> GetKind() == TokenKind::AssignmentToken);
}
TEST_CASE( "Lex Equality Token", "[lexer]" ) {
Lexer lexer = Lexer("==", nullptr);
Lexer lexer = Lexer(u"==", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@@ -65,7 +67,7 @@ TEST_CASE( "Lex Equality Token", "[lexer]" ) {
}
TEST_CASE( "Lex Whitespace", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
CHECK(lexer.LexNext(' ') -> GetKind() == TokenKind::WhiteSpace);
CHECK(lexer.LexNext('\t') -> GetKind() == TokenKind::WhiteSpace);
CHECK(lexer.LexNext('\n') -> GetKind() == TokenKind::WhiteSpace);
@@ -75,7 +77,7 @@ TEST_CASE( "Lex Whitespace", "[lexer]" ) {
}
TEST_CASE( "Lex Basic Digits", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
CHECK(lexer.LexNext('0') -> GetKind() == TokenKind::Integer);
CHECK(lexer.LexNext('1') -> GetKind() == TokenKind::Integer);
CHECK(lexer.LexNext('2') -> GetKind() == TokenKind::Integer);
@@ -88,10 +90,16 @@ TEST_CASE( "Lex Basic Digits", "[lexer]" ) {
CHECK(lexer.LexNext('9') -> GetKind() == TokenKind::Integer);
}
std::u16string to_u16string(long const &i) {
std::wstring_convert<std::codecvt_utf8_utf16<char16_t, 0x10ffff, std::little_endian>, char16_t> conv;
return conv.from_bytes(std::to_string(i));
}
TEST_CASE( "Lex Longer Integers", "[lexer]" ) {
long integers[] {0,1,5,9,10,50,100,1000,99999,6484,62163,48862};
for (long integer : integers){
Lexer lexer = Lexer(std::to_string(integer), nullptr);
auto s = to_u16string(integer);
Lexer lexer = Lexer(s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@@ -101,10 +109,15 @@ TEST_CASE( "Lex Longer Integers", "[lexer]" ) {
}
}
std::u16string to_u16string(double const &i) {
std::wstring_convert<std::codecvt_utf8_utf16<char16_t, 0x10ffff, std::little_endian>, char16_t> conv;
return conv.from_bytes(std::to_string(i));}
TEST_CASE( "Lex Floats", "[lexer]" ) {
double floats[] {0.5, 0.8, 100.7, 52.3548, 8461354.1324886};
for (double f : floats){
Lexer lexer = Lexer(std::to_string(f), nullptr);
auto s = to_u16string(f);
Lexer lexer = Lexer(s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@@ -115,133 +128,134 @@ TEST_CASE( "Lex Floats", "[lexer]" ) {
}
TEST_CASE( "Lex And Keyword", "[lexer]" ) {
Lexer lexer = Lexer("and", nullptr);
Lexer lexer = Lexer(u"and", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::AndKeyword);
}
TEST_CASE( "Lex Break Keyword", "[lexer]" ) {
Lexer lexer = Lexer("break", nullptr);
Lexer lexer = Lexer(u"break", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::BreakKeyword);
}
TEST_CASE( "Lex Do Keyword", "[lexer]" ) {
Lexer lexer = Lexer("do", nullptr);
Lexer lexer = Lexer(u"do", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::DoKeyword);
}
TEST_CASE( "Lex else Keyword", "[lexer]" ) {
Lexer lexer = Lexer("else", nullptr);
Lexer lexer = Lexer(u"else", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ElseKeyword);
}
TEST_CASE( "Lex else if Keyword", "[lexer]" ) {
Lexer lexer = Lexer("elseif", nullptr);
Lexer lexer = Lexer(u"elseif", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ElseIfKeyword);
}
TEST_CASE( "Lex end Keyword", "[lexer]" ) {
Lexer lexer = Lexer("end", nullptr);
Lexer lexer = Lexer(u"end", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::EndKeyword);
}
TEST_CASE( "Lex false Keyword", "[lexer]" ) {
Lexer lexer = Lexer("false", nullptr);
Lexer lexer = Lexer(u"false", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::FalseKeyword);
}
TEST_CASE( "Lex for Keyword", "[lexer]" ) {
Lexer lexer = Lexer("for", nullptr);
Lexer lexer = Lexer(u"for", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ForKeyword);
}
TEST_CASE( "Lex function Keyword", "[lexer]" ) {
Lexer lexer = Lexer("function", nullptr);
auto s = new u16string(u"function");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::FunctionKeyword);
}
TEST_CASE( "Lex if Keyword", "[lexer]" ) {
Lexer lexer = Lexer("if", nullptr);
Lexer lexer = Lexer(u"if", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::IfKeyword);
}
TEST_CASE( "Lex in Keyword", "[lexer]" ) {
Lexer lexer = Lexer("in", nullptr);
Lexer lexer = Lexer(u"in", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::InKeyword);
}
TEST_CASE( "Lex local Keyword", "[lexer]" ) {
Lexer lexer = Lexer("local", nullptr);
Lexer lexer = Lexer(u"local", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::LocalKeyword);
}
TEST_CASE( "Lex nil Keyword", "[lexer]" ) {
Lexer lexer = Lexer("nil", nullptr);
Lexer lexer = Lexer(u"nil", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::NilKeyword);
}
TEST_CASE( "Lex not Keyword", "[lexer]" ) {
Lexer lexer = Lexer("not", nullptr);
Lexer lexer = Lexer(u"not", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::NotKeyword);
}
TEST_CASE( "Lex or Keyword", "[lexer]" ) {
Lexer lexer = Lexer("or", nullptr);
Lexer lexer = Lexer(u"or", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::OrKeyword);
}
TEST_CASE( "Lex return Keyword", "[lexer]" ) {
Lexer lexer = Lexer("return", nullptr);
Lexer lexer = Lexer(u"return", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ReturnKeyword);
}
TEST_CASE( "Lex then Keyword", "[lexer]" ) {
Lexer lexer = Lexer("then", nullptr);
Lexer lexer = Lexer(u"then", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ThenKeyword);
}
TEST_CASE( "Lex true Keyword", "[lexer]" ) {
Lexer lexer = Lexer("true", nullptr);
Lexer lexer = Lexer(u"true", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::TrueKeyword);
}
TEST_CASE( "Lex while Keyword", "[lexer]" ) {
Lexer lexer = Lexer("while", nullptr);
Lexer lexer = Lexer(u"while", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@@ -249,7 +263,7 @@ TEST_CASE( "Lex while Keyword", "[lexer]" ) {
}
TEST_CASE( "Lex identifier", "[lexer]" ) {
Lexer lexer = Lexer("foo", nullptr);
Lexer lexer = Lexer(u"foo", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@@ -258,7 +272,8 @@ TEST_CASE( "Lex identifier", "[lexer]" ) {
}
TEST_CASE( "Lex Start Position", "[lexer]" ) {
Lexer lexer = Lexer("+ - bar 1234", nullptr);
auto s = new u16string(u"+ - bar 1234");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 5);
CHECK(((IdentifierToken*)tokens[0]) -> GetStartPosition() == 0);
@@ -269,7 +284,8 @@ TEST_CASE( "Lex Start Position", "[lexer]" ) {
}
TEST_CASE( "Lex End Position", "[lexer]" ) {
Lexer lexer = Lexer("+ - bar 1234", nullptr);
auto s = new u16string(u"+ - bar 1234");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 5);
CHECK(((IdentifierToken*)tokens[0]) -> GetEndPosition() == 0);
@@ -280,39 +296,43 @@ TEST_CASE( "Lex End Position", "[lexer]" ) {
}
TEST_CASE("Lex Double Quote String", "[lexer]") {
Lexer lexer = Lexer("\"foo bar\"", nullptr);
auto s = new u16string(u"\"foo bar\"");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo bar");
}
TEST_CASE("Lex Single Quote String", "[lexer]") {
Lexer lexer = Lexer("'foo bar'", nullptr);
auto s = new u16string(u"'foo bar'");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo bar");
}
TEST_CASE("Lex Double Quote String, Escape Quote", "[lexer]") {
Lexer lexer = Lexer("'foo\\\"bar'", nullptr);
auto s = new u16string(u"'foo\\\"bar'");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\"bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo\"bar");
}
TEST_CASE("Lex String with newline", "[lexer]") {
Lexer lexer = Lexer("'foo\\nbar'", nullptr);
auto s = new u16string(u"'foo\\nbar'");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\nbar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo\nbar");
}

4
tests/parser/ParserTests.cpp
View File

@@ -142,7 +142,7 @@ TEST_CASE( "Assert binary precedence", "[parser]" ) {
}
TEST_CASE( "Parse String Tokens", "[parser]" ) {
vector<const IToken*> v {new StringToken("foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
vector<const IToken*> v {new StringToken(u"foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
Parser parser = Parser(v, nullptr);
auto parsedStatements = parser.Parse() -> GetStatements();
REQUIRE(parsedStatements->size() == 1);
@@ -151,7 +151,7 @@ TEST_CASE( "Parse String Tokens", "[parser]" ) {
auto expression = ((ParsedExpressionStatement*)firstStatement)->GetExpression();
REQUIRE(expression -> GetKind() == ParsedExpressionKind::LiteralString);
auto boolean = ((LiteralStringExpression*)expression);
REQUIRE(boolean->GetValue() == "foo bar");
REQUIRE(boolean->GetValue() == u"foo bar");
}
TEST_CASE( "Parse Global Assignment", "[parser]" ) {