Deukhoofd
/
PorygonLang
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move Lexer to u16string handling, for unicode support
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Deukhoofd 2019-06-15 17:20:27 +02:00
parent f73bd2003c
commit 3dc67ec8a0
No known key found for this signature in database
GPG Key ID: B4C087AC81641654
21 changed files with 189 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/Binder/BoundExpressions/BoundExpression.hpp
View File

@ -104,9 +104,9 @@ public:
};
class BoundLiteralStringExpression : public BoundExpression{
const string _value;
const u16string _value;
public:
BoundLiteralStringExpression(string value, unsigned int start, unsigned int length)
BoundLiteralStringExpression(u16string value, unsigned int start, unsigned int length)
: BoundExpression(start, length, make_shared<StringScriptType>(true, HashedString::ConstHash(value.c_str()))),
_value(value)
{
@ -116,7 +116,7 @@ public:
return BoundExpressionKind ::LiteralString;
}
const string GetValue() const{
const u16string GetValue() const{
return _value;
}
};

2
src/Evaluator/BinaryEvaluation.cpp
View File

@ -96,7 +96,7 @@ shared_ptr<BooleanEvalValue> Evaluator::EvaluateBooleanBinary(const BoundBinaryE
shared_ptr<StringEvalValue> Evaluator::EvaluateStringBinary(const BoundBinaryExpression* expression){
if (expression->GetOperation() != BoundBinaryOperation::Concatenation)
throw;
std::ostringstream strs;
std::basic_ostringstream<char16_t > strs;
auto left = this -> EvaluateStringExpression(expression->GetLeft());
strs << *left->EvaluateString();
auto right = this -> EvaluateExpression(expression->GetRight());

9
src/Evaluator/EvalValues/EvalValue.cpp
View File

@ -20,7 +20,7 @@ extern "C" {
return v->EvaluateBool();
}
const char* EvaluateEvalValueString(EvalValue* v){
const char16_t * EvaluateEvalValueString(EvalValue* v){
return v->EvaluateString() -> c_str();
}
@ -36,7 +36,7 @@ extern "C" {
return new BooleanEvalValue(b);
}
EvalValue* CreateStringEvalValue(const char* s){
EvalValue* CreateStringEvalValue(const char16_t * s){
return new StringEvalValue(s);
}
}
@ -47,11 +47,12 @@ extern "C" {
TEST_CASE( "Evaluate String", "[integration]" ) {
auto script = Script::Create("\"foo bar\"");
auto script = Script::Create(u"\"foo bar\"");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(std::strcmp(EvaluateEvalValueString(lastValue), "foo bar") == 0);
auto s = u16string(EvaluateEvalValueString(lastValue));
REQUIRE(s == u"foo bar");
delete script;
}

2
src/Evaluator/EvalValues/EvalValue.hpp
View File

@ -31,7 +31,7 @@ public:
virtual bool EvaluateBool() const{
throw EvaluationException("Can't evaluate this EvalValue as bool.");
}
virtual const std::string* EvaluateString() const {
virtual const std::u16string* EvaluateString() const {
throw EvaluationException("Can't evaluate this EvalValue as string.");
}

8
src/Evaluator/EvalValues/StringEvalValue.hpp
View File

@ -9,10 +9,10 @@
using namespace std;
class StringEvalValue : public EvalValue{
string _value;
u16string _value;
size_t _hash;
public:
explicit StringEvalValue(string s){
explicit StringEvalValue(u16string s){
_value = move(s);
_hash = HashedString::ConstHash (_value.c_str());
}
@ -27,7 +27,7 @@ public:
return this->_hash == b->GetHashCode();
};
const string* EvaluateString() const final{
const u16string* EvaluateString() const final{
return &_value;
}
@ -38,7 +38,7 @@ public:
shared_ptr<EvalValue> IndexValue(EvalValue* val) final{
// Porygon is 1-indexed, so we convert to that.
auto l = val->EvaluateInteger() - 1;
return make_shared<StringEvalValue>(string(1, _value[l]));
return make_shared<StringEvalValue>(u16string(1, _value[l]));
}
std::size_t GetHashCode() final{

32
src/Parser/Lexer.cpp
View File

@ -5,7 +5,7 @@
#include "Lexer.hpp"
Lexer::Lexer(const string& scriptString, class Script* script)
Lexer::Lexer(const u16string& scriptString, class Script* script)
: _scriptString(scriptString)
{
this->_scriptSize = scriptString.size();
@ -29,19 +29,19 @@ vector<const IToken*> Lexer::Lex() {
return tokens;
}
char Lexer::Peek(){
char16_t Lexer::Peek(){
if (Lexer::_position >= this -> _scriptSize)
return '\0';
return this -> _scriptString.at(Lexer::_position);
}
char Lexer::Next(){
char next = Peek();
char16_t Lexer::Next(){
char16_t next = Peek();
Lexer::_position++;
return next;
}
IToken* Lexer::LexNext(char c){
IToken* Lexer::LexNext(char16_t c){
switch (c) {
case '\0':
return new SimpleToken(TokenKind::EndOfFile, this -> _position - 1, 1);
@ -113,7 +113,7 @@ IToken* Lexer::LexNext(char c){
}
}
int CharToInt(char c){
int CharToInt(char16_t c){
switch (c){
case '0': return 0;
case '1': return 1;
@ -129,7 +129,7 @@ int CharToInt(char c){
}
}
IToken* Lexer::LexNumber(char c){
IToken* Lexer::LexNumber(char16_t c){
long int_value = CharToInt(c);
double float_value = 0;
short decimal_index = 0;
@ -138,7 +138,7 @@ IToken* Lexer::LexNumber(char c){
unsigned int start = this -> _position - 1;
unsigned int length = 1;
while (is_searching){
char next = this -> Peek();
char16_t next = this -> Peek();
int next_val = CharToInt(next);
if (next_val == -1){
switch (next){
@ -183,7 +183,7 @@ IToken * Lexer::LexIdentifierOrKeyword() {
auto start = this -> _position - 1;
auto end = start;
while (true){
char next = this -> Peek();
char16_t next = this -> Peek();
if (next == '\0') break;
if (isalpha(next) || next == '_'){
this -> Next();
@ -194,7 +194,7 @@ IToken * Lexer::LexIdentifierOrKeyword() {
}
}
string s = this -> _scriptString.substr(start, end - start + 1);
u16string s = this -> _scriptString.substr(start, end - start + 1);
switch (HashedString::ConstHash(s.c_str())){
case HashedString::ConstHash("and"): return new SimpleToken(TokenKind::AndKeyword, start, 3);
case HashedString::ConstHash("break"): return new SimpleToken(TokenKind::BreakKeyword, start, 5);
@ -219,7 +219,7 @@ IToken * Lexer::LexIdentifierOrKeyword() {
}
}
const unordered_map<char, char> ControlCharacters{ // NOLINT(cert-err58-cpp)
const unordered_map<char16_t, char16_t> ControlCharacters{ // NOLINT(cert-err58-cpp)
{'0', '\0'},
{'a', '\a'},
{'b', '\b'},
@ -234,12 +234,12 @@ const unordered_map<char, char> ControlCharacters{ // NOLINT(cert-err58-cpp)
{'\\', '\\'},
};
IToken* Lexer::LexString(char c){
IToken* Lexer::LexString(char16_t c){
auto start = this -> _position - 1;
auto end = start;
char last = c;
char16_t last = c;
while (true){
char next = this -> Peek();
char16_t next = this -> Peek();
if (next == '\0') break;
if (next == c && last != '\\') break;
this -> Next();
@ -252,8 +252,8 @@ IToken* Lexer::LexString(char c){
return new SimpleToken(TokenKind::BadToken, start, end -start + 1);
}
string s = this -> _scriptString.substr(start + 1, end - start);
stringstream stream;
u16string s = this -> _scriptString.substr(start + 1, end - start);
std::basic_ostringstream<char16_t > stream;
for (int i = 0; i < s.size(); i++){
c = s[i];
if (c == '\\'){

14
src/Parser/Lexer.hpp
View File

@ -8,23 +8,23 @@
using namespace std;
class Lexer {
const string& _scriptString;
const u16string& _scriptString;
#ifdef TESTS_BUILD
public:
#endif
unsigned int _position;
unsigned int _scriptSize;
char Peek();
char Next();
IToken* LexNext(char c);
IToken* LexNumber(char c);
char16_t Peek();
char16_t Next();
IToken* LexNext(char16_t c);
IToken* LexNumber(char16_t c);
IToken* LexIdentifierOrKeyword();
IToken* LexString(char c);
IToken* LexString(char16_t c);
public:
Script* ScriptData;
vector<const IToken*> Lex();
explicit Lexer(const string& scriptString, class Script* script);
explicit Lexer(const u16string& scriptString, class Script* script);
};

4
src/Parser/ParsedExpressions/ParsedExpression.hpp
View File

@ -100,7 +100,7 @@ public:
};
class LiteralStringExpression : public ParsedExpression{
const string _value;
const u16string _value;
public:
const ParsedExpressionKind GetKind() const final{
return ParsedExpressionKind::LiteralString;
@ -111,7 +111,7 @@ public:
{
}
const string& GetValue() const{
const u16string& GetValue() const{
return _value;
}
};

6
src/Parser/Token.hpp
View File

@ -91,10 +91,10 @@ public:
};
class StringToken : public IToken{
const string _value;
const u16string _value;
public:
explicit StringToken(string value, unsigned int position, unsigned int length)
explicit StringToken(u16string value, unsigned int position, unsigned int length)
: IToken(position, length),
_value(std::move(value))
{
@ -104,7 +104,7 @@ public:
return TokenKind::String;
}
const string& GetValue() const{
const u16string& GetValue() const{
return _value;
}
};

34
src/Script.cpp
View File

@ -1,18 +1,29 @@
#include <utility>
#include <vector>
#include <iterator>
#include <locale>
#include <unordered_map>
#include <codecvt>
#include "Script.hpp"
#include "Parser/Lexer.hpp"
#include "Parser/Parser.hpp"
#include "Binder/Binder.hpp"
Script* Script::Create(const string& script) {
Script* Script::Create(const u16string& script) {
auto s = new Script();
s -> Parse(script);
return s;
}
std::u16string To_UTF16(const string &s)
{
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> conv;
return conv.from_bytes(s);
}
Script *Script::Create(const string &script) {
return Script::Create(To_UTF16(script));
}
Script::Script() {
Diagnostics = new DiagnosticsHolder();
_evaluator = new Evaluator(this);
@ -32,7 +43,7 @@ Script::~Script() {
delete this->_scriptVariables;
}
void Script::Parse(const string& script) {
void Script::Parse(const u16string& script) {
auto lexer = Lexer(script, this);
auto lexResult = lexer.Lex();
auto parser = Parser(lexResult, this);
@ -54,11 +65,11 @@ void Script::Parse(const string& script) {
delete parseResult;
}
EvalValue *Script::GetVariable(const string &key) {
EvalValue *Script::GetVariable(const u16string &key) {
return _scriptVariables -> at(HashedString(key).GetHash()).get();
}
bool Script::HasVariable(const string &key) {
bool Script::HasVariable(const u16string &key) {
auto f = _scriptVariables->find(HashedString(key).GetHash());
return f != _scriptVariables->end();
}
@ -67,18 +78,19 @@ EvalValue *Script::GetLastValue() {
return _evaluator->GetLastValue();
}
bool Script::HasFunction(const string &key) {
bool Script::HasFunction(const u16string &key) {
auto f = _scriptVariables->find(HashedString(key).GetHash());
return f != _scriptVariables->end() && f.operator->()->second->GetTypeClass() == TypeClass ::Function;
}
shared_ptr<EvalValue> Script::CallFunction(const string &key, const vector<EvalValue *>& variables) {
shared_ptr<EvalValue> Script::CallFunction(const u16string &key, const vector<EvalValue *>& variables) {
auto var = (ScriptFunctionEvalValue*)GetVariable(key);
return this->_evaluator->EvaluateFunction(var, variables);
}
extern "C" {
Script* CreateScript(char * s){
Script* CreateScript(char16_t * s){
return Script::Create(s);
}
@ -90,19 +102,19 @@ extern "C" {
return script->GetLastValue();
}
bool HasVariable(Script* script, const char* key){
bool HasVariable(Script* script, const char16_t* key){
return script->HasVariable(key);
}
EvalValue* GetVariable(Script* script, const char* key){
EvalValue* GetVariable(Script* script, const char16_t* key){
return script->GetVariable(key);
}
bool HasFunction(Script* script, const char* key){
bool HasFunction(Script* script, const char16_t* key){
return script->HasFunction(key);
}
EvalValue* CallFunction(Script* script, const char* key, EvalValue* parameters[], int parameterCount){
EvalValue* CallFunction(Script* script, const char16_t* key, EvalValue* parameters[], int parameterCount){
std::vector<EvalValue*> v(parameters, parameters + parameterCount);
return script->CallFunction(key, v).get();
}

11
src/Script.hpp
View File

@ -23,8 +23,9 @@ class Script {
shared_ptr<ScriptType> _returnType;
explicit Script();
void Parse(const string& script);
void Parse(const u16string& script);
public:
static Script* Create(const u16string& script);
static Script* Create(const string& script);
DiagnosticsHolder* Diagnostics;
@ -42,11 +43,11 @@ public:
EvalValue* GetLastValue();
EvalValue* GetVariable(const string& key);
bool HasVariable(const string& key);
EvalValue* GetVariable(const u16string& key);
bool HasVariable(const u16string& key);
shared_ptr<EvalValue> CallFunction(const string& key, const vector<EvalValue*>& variables);
bool HasFunction(const string& key);
shared_ptr<EvalValue> CallFunction(const u16string& key, const vector<EvalValue*>& variables);
bool HasFunction(const u16string& key);
};

12
src/Utilities/HashedString.hpp
View File

@ -7,12 +7,22 @@
class HashedString{
const uint32_t _hash;
public:
explicit HashedString(const std::string& s) : _hash(ConstHash(s.c_str())){
explicit HashedString(const std::u16string& s) : _hash(ConstHash(s.c_str())){
}
explicit HashedString(char16_t const *input) : _hash(ConstHash(input)){
}
explicit HashedString(char const *input) : _hash(ConstHash(input)){
}
HashedString(const HashedString& b) = default;
static uint32_t constexpr ConstHash(char16_t const *input) {
return *input ?
static_cast<uint32_t>(*input) + 33 * ConstHash(input + 1) :
5381;
}
static uint32_t constexpr ConstHash(char const *input) {
return *input ?
static_cast<uint32_t>(*input) + 33 * ConstHash(input + 1) :

12
tests/integration/ConditionalTests.cpp
View File

@ -5,10 +5,10 @@
TEST_CASE( "Basic conditional", "[integration]" ) {
Script* script = Script::Create("if true then foo = true end");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;
@ -17,10 +17,10 @@ TEST_CASE( "Basic conditional", "[integration]" ) {
TEST_CASE( "If then, else", "[integration]" ) {
Script* script = Script::Create("if false then foo = false else foo = true end");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;
@ -29,10 +29,10 @@ TEST_CASE( "If then, else", "[integration]" ) {
TEST_CASE( "If then, else if", "[integration]" ) {
Script* script = Script::Create("if false then foo = false elseif true then foo = true end");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;

30
tests/integration/Functions.cpp
View File

@ -6,7 +6,7 @@ TEST_CASE( "Define script function", "[integration]" ) {
Script* script = Script::Create("function add(number a, number b) a + b end");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("add");
auto variable = script->GetVariable(u"add");
REQUIRE(variable != nullptr);
REQUIRE(variable->GetTypeClass() == TypeClass::Function);
delete script;
@ -16,10 +16,10 @@ TEST_CASE( "Define script function and call", "[integration]" ) {
Script* script = Script::Create("function add(number a, number b) result = a + b end add(1, 2)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("add");
auto variable = script->GetVariable(u"add");
REQUIRE(variable != nullptr);
REQUIRE(variable->GetTypeClass() == TypeClass::Function);
auto result = script->GetVariable("result");
auto result = script->GetVariable(u"result");
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 3);
delete script;
@ -29,10 +29,10 @@ TEST_CASE( "Define script function and call multiple times", "[integration]" ) {
Script* script = Script::Create("result = 0 function add(number a) result = result + a end add(1) add(4)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("add");
auto variable = script->GetVariable(u"add");
REQUIRE(variable != nullptr);
REQUIRE(variable->GetTypeClass() == TypeClass::Function);
auto result = script->GetVariable("result");
auto result = script->GetVariable(u"result");
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 5);
delete script;
@ -43,15 +43,15 @@ TEST_CASE( "Define script function and call from extern", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
REQUIRE(script->HasFunction("add"));
REQUIRE(script->HasFunction(u"add"));
auto toAddVal = new IntegerEvalValue(5);
script->CallFunction("add", {toAddVal});
script->CallFunction(u"add", {toAddVal});
delete toAddVal;
toAddVal = new IntegerEvalValue(6);
script->CallFunction("add", {toAddVal});
script->CallFunction(u"add", {toAddVal});
delete toAddVal;
auto result = script->GetVariable("result");
auto result = script->GetVariable(u"result");
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 11);
delete script;
@ -67,17 +67,17 @@ TEST_CASE( "Define script function and return", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
REQUIRE(script->HasFunction("add"));
REQUIRE(script->HasFunction(u"add"));
auto toAddVal = new IntegerEvalValue(5);
auto toAddVal2 = new IntegerEvalValue(6);
auto result = script->CallFunction("add", {toAddVal, toAddVal2});
auto result = script->CallFunction(u"add", {toAddVal, toAddVal2});
delete toAddVal;
delete toAddVal2;
REQUIRE(result->GetTypeClass() == TypeClass::Number);
REQUIRE(result->EvaluateInteger() == 11);
auto variable = script->GetVariable("val");
auto variable = script->GetVariable(u"val");
REQUIRE(variable->GetTypeClass() == TypeClass::Number);
REQUIRE(variable->EvaluateInteger() == 0);
@ -97,9 +97,9 @@ end
)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
script->CallFunction("add", {});
script->CallFunction(u"add", {});
auto variable = script->GetVariable("val");
auto variable = script->GetVariable(u"val");
REQUIRE(variable->GetTypeClass() == TypeClass::Number);
REQUIRE(variable->EvaluateInteger() == 5);
@ -127,7 +127,7 @@ test()
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("result");
auto variable = script->GetVariable(u"result");
REQUIRE(variable->GetTypeClass() == TypeClass::Number);
REQUIRE(variable->EvaluateInteger() == 50);

2
tests/integration/IndexTests.cpp
View File

@ -7,7 +7,7 @@ TEST_CASE( "String indexing", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "b");
REQUIRE(*lastValue->EvaluateString() == u"b");
delete script;
}

6
tests/integration/StringOperationsTests.cpp
View File

@ -9,7 +9,7 @@ TEST_CASE( "Simple String", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "foo bar");
REQUIRE(*lastValue->EvaluateString() == u"foo bar");
delete script;
}
@ -18,7 +18,7 @@ TEST_CASE( "String Concat", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "foobar");
REQUIRE(*lastValue->EvaluateString() == u"foobar");
delete script;
}
@ -27,7 +27,7 @@ TEST_CASE( "String Concat 2", "[integration]" ) {
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto lastValue = script->GetLastValue();
REQUIRE(*lastValue->EvaluateString() == "foobar");
REQUIRE(*lastValue->EvaluateString() == u"foobar");
delete script;
}

12
tests/integration/Tables.cpp
View File

@ -7,7 +7,7 @@ TEST_CASE( "Create empty table", "[integration]" ) {
Script* script = Script::Create("table = {}");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
delete script;
}
@ -16,7 +16,7 @@ TEST_CASE( "Create simple integer table", "[integration]" ) {
Script* script = Script::Create("table = {100, 50, 20, 5, -100, 50+50}");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
delete script;
}
@ -25,7 +25,7 @@ TEST_CASE( "Create simple string table", "[integration]" ) {
Script* script = Script::Create("table = {'bla', 'test', 'foo', 'bar'}");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
delete script;
}
@ -39,7 +39,7 @@ return table[3]
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->Evaluate();
REQUIRE(variable != nullptr);
REQUIRE(*variable->EvaluateString() == "foo");
REQUIRE(*variable->EvaluateString() == u"foo");
delete script;
}
@ -53,10 +53,10 @@ table = {
)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("table");
auto variable = script->GetVariable(u"table");
REQUIRE(variable != nullptr);
auto table = (TableEvalValue*)variable;
CHECK(*table->IndexValue("foo")->EvaluateString() == "test");
CHECK(*table->IndexValue("foo")->EvaluateString() == u"test");
CHECK(table->IndexValue("bar")->EvaluateInteger() == 100);
delete script;
}

4
tests/integration/UserData.cpp
View File

@ -38,7 +38,7 @@ end
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto parameter = new UserDataValue(HashedString::ConstHash("testObject"), new UserDataTestObject());
auto variable = script->CallFunction("testFunc", {parameter});
auto variable = script->CallFunction(u"testFunc", {parameter});
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateInteger() == 10);
delete script;
@ -55,7 +55,7 @@ end
script->Evaluate();
auto obj = new UserDataTestObject();
auto parameter = new UserDataValue(HashedString::ConstHash("testObject"), obj);
script->CallFunction("testFunc", {parameter});
script->CallFunction(u"testFunc", {parameter});
delete script;
REQUIRE(obj->foo == 5000);
delete obj;

16
tests/integration/Variables.cpp
View File

@ -5,10 +5,10 @@
TEST_CASE( "Create script variable", "[integration]" ) {
Script* script = Script::Create("foo = true");
REQUIRE(!script->Diagnostics -> HasErrors());
auto variable = script->GetVariable("foo");
auto variable = script->GetVariable(u"foo");
REQUIRE(variable == nullptr);
script->Evaluate();
variable = script->GetVariable("foo");
variable = script->GetVariable(u"foo");
REQUIRE(variable != nullptr);
REQUIRE(variable->EvaluateBool());
delete script;
@ -17,7 +17,7 @@ TEST_CASE( "Create script variable", "[integration]" ) {
TEST_CASE( "Create local variable", "[integration]" ) {
Script* script = Script::Create("local foo = true");
REQUIRE(!script->Diagnostics -> HasErrors());
REQUIRE_FALSE(script->HasVariable("foo"));
REQUIRE_FALSE(script->HasVariable(u"foo"));
script->Evaluate();
delete script;
}
@ -27,7 +27,7 @@ TEST_CASE( "Create script variable and use", "[integration]" ) {
"bar = not foo");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("bar");
auto variable = script->GetVariable(u"bar");
REQUIRE(variable != nullptr);
CHECK(variable->EvaluateBool());
delete script;
@ -38,7 +38,7 @@ TEST_CASE( "Create local variable and use", "[integration]" ) {
"bar = not foo");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
auto variable = script->GetVariable("bar");
auto variable = script->GetVariable(u"bar");
REQUIRE(variable != nullptr);
CHECK(variable->EvaluateBool());
delete script;
@ -55,9 +55,9 @@ end
)");
REQUIRE(!script->Diagnostics -> HasErrors());
script->Evaluate();
script -> CallFunction("bar", {});
script -> CallFunction("bar", {});
auto variable = script->GetVariable("result");
script -> CallFunction(u"bar", {});
script -> CallFunction(u"bar", {});
auto variable = script->GetVariable(u"result");
REQUIRE(variable != nullptr);
CHECK(variable->EvaluateInteger() == 2);
delete script;

108
tests/parser/LexerTests.cpp
View File

@ -1,14 +1,16 @@
#ifdef TESTS_BUILD
#include <catch.hpp>
#include <codecvt>
#include <locale>
#include "../../src/Parser/Lexer.hpp"
TEST_CASE( "When at end of script return terminator", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.Peek() == '\0');
}
TEST_CASE( "Peek doesn't advance", "[lexer]" ) {
auto script = new string("5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
auto script = new u16string(u"5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
Lexer lexer = Lexer(*script, nullptr);
REQUIRE(lexer.Peek() == '5');
REQUIRE(lexer.Peek() == '5');
@ -16,7 +18,7 @@ TEST_CASE( "Peek doesn't advance", "[lexer]" ) {
}
TEST_CASE( "Next does advance", "[lexer]" ) {
auto script = new string("5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
auto script = new u16string(u"5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds
Lexer lexer = Lexer(*script, nullptr);
REQUIRE(lexer.Next() == '5');
REQUIRE(lexer.Next() == ' ');
@ -27,37 +29,37 @@ TEST_CASE( "Next does advance", "[lexer]" ) {
}
TEST_CASE( "Lex Null Terminator as EOF", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('\0') -> GetKind() == TokenKind::EndOfFile);
}
TEST_CASE( "Lex Plus Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('+') -> GetKind() == TokenKind::PlusToken);
}
TEST_CASE( "Lex Minus Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('-') -> GetKind() == TokenKind::MinusToken);
}
TEST_CASE( "Lex Slash Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('/') -> GetKind() == TokenKind::SlashToken);
}
TEST_CASE( "Lex Star Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('*') -> GetKind() == TokenKind::StarToken);
}
TEST_CASE( "Lex Assignment Token", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
REQUIRE(lexer.LexNext('=') -> GetKind() == TokenKind::AssignmentToken);
}
TEST_CASE( "Lex Equality Token", "[lexer]" ) {
Lexer lexer = Lexer("==", nullptr);
Lexer lexer = Lexer(u"==", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@ -65,7 +67,7 @@ TEST_CASE( "Lex Equality Token", "[lexer]" ) {
}
TEST_CASE( "Lex Whitespace", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
CHECK(lexer.LexNext(' ') -> GetKind() == TokenKind::WhiteSpace);
CHECK(lexer.LexNext('\t') -> GetKind() == TokenKind::WhiteSpace);
CHECK(lexer.LexNext('\n') -> GetKind() == TokenKind::WhiteSpace);
@ -75,7 +77,7 @@ TEST_CASE( "Lex Whitespace", "[lexer]" ) {
}
TEST_CASE( "Lex Basic Digits", "[lexer]" ) {
Lexer lexer = Lexer("", nullptr);
Lexer lexer = Lexer(u"", nullptr);
CHECK(lexer.LexNext('0') -> GetKind() == TokenKind::Integer);
CHECK(lexer.LexNext('1') -> GetKind() == TokenKind::Integer);
CHECK(lexer.LexNext('2') -> GetKind() == TokenKind::Integer);
@ -88,10 +90,16 @@ TEST_CASE( "Lex Basic Digits", "[lexer]" ) {
CHECK(lexer.LexNext('9') -> GetKind() == TokenKind::Integer);
}
std::u16string to_u16string(long const &i) {
std::wstring_convert<std::codecvt_utf8_utf16<char16_t, 0x10ffff, std::little_endian>, char16_t> conv;
return conv.from_bytes(std::to_string(i));
}
TEST_CASE( "Lex Longer Integers", "[lexer]" ) {
long integers[] {0,1,5,9,10,50,100,1000,99999,6484,62163,48862};
for (long integer : integers){
Lexer lexer = Lexer(std::to_string(integer), nullptr);
auto s = to_u16string(integer);
Lexer lexer = Lexer(s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@ -101,10 +109,15 @@ TEST_CASE( "Lex Longer Integers", "[lexer]" ) {
}
}
std::u16string to_u16string(double const &i) {
std::wstring_convert<std::codecvt_utf8_utf16<char16_t, 0x10ffff, std::little_endian>, char16_t> conv;
return conv.from_bytes(std::to_string(i));}
TEST_CASE( "Lex Floats", "[lexer]" ) {
double floats[] {0.5, 0.8, 100.7, 52.3548, 8461354.1324886};
for (double f : floats){
Lexer lexer = Lexer(std::to_string(f), nullptr);
auto s = to_u16string(f);
Lexer lexer = Lexer(s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@ -115,133 +128,134 @@ TEST_CASE( "Lex Floats", "[lexer]" ) {
}
TEST_CASE( "Lex And Keyword", "[lexer]" ) {
Lexer lexer = Lexer("and", nullptr);
Lexer lexer = Lexer(u"and", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::AndKeyword);
}
TEST_CASE( "Lex Break Keyword", "[lexer]" ) {
Lexer lexer = Lexer("break", nullptr);
Lexer lexer = Lexer(u"break", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::BreakKeyword);
}
TEST_CASE( "Lex Do Keyword", "[lexer]" ) {
Lexer lexer = Lexer("do", nullptr);
Lexer lexer = Lexer(u"do", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::DoKeyword);
}
TEST_CASE( "Lex else Keyword", "[lexer]" ) {
Lexer lexer = Lexer("else", nullptr);
Lexer lexer = Lexer(u"else", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ElseKeyword);
}
TEST_CASE( "Lex else if Keyword", "[lexer]" ) {
Lexer lexer = Lexer("elseif", nullptr);
Lexer lexer = Lexer(u"elseif", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ElseIfKeyword);
}
TEST_CASE( "Lex end Keyword", "[lexer]" ) {
Lexer lexer = Lexer("end", nullptr);
Lexer lexer = Lexer(u"end", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::EndKeyword);
}
TEST_CASE( "Lex false Keyword", "[lexer]" ) {
Lexer lexer = Lexer("false", nullptr);
Lexer lexer = Lexer(u"false", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::FalseKeyword);
}
TEST_CASE( "Lex for Keyword", "[lexer]" ) {
Lexer lexer = Lexer("for", nullptr);
Lexer lexer = Lexer(u"for", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ForKeyword);
}
TEST_CASE( "Lex function Keyword", "[lexer]" ) {
Lexer lexer = Lexer("function", nullptr);
auto s = new u16string(u"function");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::FunctionKeyword);
}
TEST_CASE( "Lex if Keyword", "[lexer]" ) {
Lexer lexer = Lexer("if", nullptr);
Lexer lexer = Lexer(u"if", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::IfKeyword);
}
TEST_CASE( "Lex in Keyword", "[lexer]" ) {
Lexer lexer = Lexer("in", nullptr);
Lexer lexer = Lexer(u"in", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::InKeyword);
}
TEST_CASE( "Lex local Keyword", "[lexer]" ) {
Lexer lexer = Lexer("local", nullptr);
Lexer lexer = Lexer(u"local", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::LocalKeyword);
}
TEST_CASE( "Lex nil Keyword", "[lexer]" ) {
Lexer lexer = Lexer("nil", nullptr);
Lexer lexer = Lexer(u"nil", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::NilKeyword);
}
TEST_CASE( "Lex not Keyword", "[lexer]" ) {
Lexer lexer = Lexer("not", nullptr);
Lexer lexer = Lexer(u"not", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::NotKeyword);
}
TEST_CASE( "Lex or Keyword", "[lexer]" ) {
Lexer lexer = Lexer("or", nullptr);
Lexer lexer = Lexer(u"or", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::OrKeyword);
}
TEST_CASE( "Lex return Keyword", "[lexer]" ) {
Lexer lexer = Lexer("return", nullptr);
Lexer lexer = Lexer(u"return", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ReturnKeyword);
}
TEST_CASE( "Lex then Keyword", "[lexer]" ) {
Lexer lexer = Lexer("then", nullptr);
Lexer lexer = Lexer(u"then", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::ThenKeyword);
}
TEST_CASE( "Lex true Keyword", "[lexer]" ) {
Lexer lexer = Lexer("true", nullptr);
Lexer lexer = Lexer(u"true", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::TrueKeyword);
}
TEST_CASE( "Lex while Keyword", "[lexer]" ) {
Lexer lexer = Lexer("while", nullptr);
Lexer lexer = Lexer(u"while", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@ -249,7 +263,7 @@ TEST_CASE( "Lex while Keyword", "[lexer]" ) {
}
TEST_CASE( "Lex identifier", "[lexer]" ) {
Lexer lexer = Lexer("foo", nullptr);
Lexer lexer = Lexer(u"foo", nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
@ -258,7 +272,8 @@ TEST_CASE( "Lex identifier", "[lexer]" ) {
}
TEST_CASE( "Lex Start Position", "[lexer]" ) {
Lexer lexer = Lexer("+ - bar 1234", nullptr);
auto s = new u16string(u"+ - bar 1234");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 5);
CHECK(((IdentifierToken*)tokens[0]) -> GetStartPosition() == 0);
@ -269,7 +284,8 @@ TEST_CASE( "Lex Start Position", "[lexer]" ) {
}
TEST_CASE( "Lex End Position", "[lexer]" ) {
Lexer lexer = Lexer("+ - bar 1234", nullptr);
auto s = new u16string(u"+ - bar 1234");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 5);
CHECK(((IdentifierToken*)tokens[0]) -> GetEndPosition() == 0);
@ -280,39 +296,43 @@ TEST_CASE( "Lex End Position", "[lexer]" ) {
}
TEST_CASE("Lex Double Quote String", "[lexer]") {
Lexer lexer = Lexer("\"foo bar\"", nullptr);
auto s = new u16string(u"\"foo bar\"");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo bar");
}
TEST_CASE("Lex Single Quote String", "[lexer]") {
Lexer lexer = Lexer("'foo bar'", nullptr);
auto s = new u16string(u"'foo bar'");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo bar");
}
TEST_CASE("Lex Double Quote String, Escape Quote", "[lexer]") {
Lexer lexer = Lexer("'foo\\\"bar'", nullptr);
auto s = new u16string(u"'foo\\\"bar'");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\"bar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo\"bar");
}
TEST_CASE("Lex String with newline", "[lexer]") {
Lexer lexer = Lexer("'foo\\nbar'", nullptr);
auto s = new u16string(u"'foo\\nbar'");
Lexer lexer = Lexer(*s, nullptr);
auto tokens = lexer.Lex();
REQUIRE(tokens.size() == 2);
const IToken* firstToken = tokens[0];
REQUIRE(firstToken -> GetKind() == TokenKind::String);
REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\nbar");
REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo\nbar");
}

4
tests/parser/ParserTests.cpp
View File

@ -142,7 +142,7 @@ TEST_CASE( "Assert binary precedence", "[parser]" ) {
}
TEST_CASE( "Parse String Tokens", "[parser]" ) {
vector<const IToken*> v {new StringToken("foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
vector<const IToken*> v {new StringToken(u"foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)};
Parser parser = Parser(v, nullptr);
auto parsedStatements = parser.Parse() -> GetStatements();
REQUIRE(parsedStatements->size() == 1);
@ -151,7 +151,7 @@ TEST_CASE( "Parse String Tokens", "[parser]" ) {
auto expression = ((ParsedExpressionStatement*)firstStatement)->GetExpression();
REQUIRE(expression -> GetKind() == ParsedExpressionKind::LiteralString);
auto boolean = ((LiteralStringExpression*)expression);
REQUIRE(boolean->GetValue() == "foo bar");
REQUIRE(boolean->GetValue() == u"foo bar");
}
TEST_CASE( "Parse Global Assignment", "[parser]" ) {