diff --git a/src/Binder/BoundExpressions/BoundExpression.hpp b/src/Binder/BoundExpressions/BoundExpression.hpp index 01fe5ff..21cbc82 100644 --- a/src/Binder/BoundExpressions/BoundExpression.hpp +++ b/src/Binder/BoundExpressions/BoundExpression.hpp @@ -104,9 +104,9 @@ public: }; class BoundLiteralStringExpression : public BoundExpression{ - const string _value; + const u16string _value; public: - BoundLiteralStringExpression(string value, unsigned int start, unsigned int length) + BoundLiteralStringExpression(u16string value, unsigned int start, unsigned int length) : BoundExpression(start, length, make_shared(true, HashedString::ConstHash(value.c_str()))), _value(value) { @@ -116,7 +116,7 @@ public: return BoundExpressionKind ::LiteralString; } - const string GetValue() const{ + const u16string GetValue() const{ return _value; } }; diff --git a/src/Evaluator/BinaryEvaluation.cpp b/src/Evaluator/BinaryEvaluation.cpp index 8715265..e32d18d 100644 --- a/src/Evaluator/BinaryEvaluation.cpp +++ b/src/Evaluator/BinaryEvaluation.cpp @@ -96,7 +96,7 @@ shared_ptr Evaluator::EvaluateBooleanBinary(const BoundBinaryE shared_ptr Evaluator::EvaluateStringBinary(const BoundBinaryExpression* expression){ if (expression->GetOperation() != BoundBinaryOperation::Concatenation) throw; - std::ostringstream strs; + std::basic_ostringstream strs; auto left = this -> EvaluateStringExpression(expression->GetLeft()); strs << *left->EvaluateString(); auto right = this -> EvaluateExpression(expression->GetRight()); diff --git a/src/Evaluator/EvalValues/EvalValue.cpp b/src/Evaluator/EvalValues/EvalValue.cpp index 375fa9b..2b78cdc 100644 --- a/src/Evaluator/EvalValues/EvalValue.cpp +++ b/src/Evaluator/EvalValues/EvalValue.cpp @@ -20,7 +20,7 @@ extern "C" { return v->EvaluateBool(); } - const char* EvaluateEvalValueString(EvalValue* v){ + const char16_t * EvaluateEvalValueString(EvalValue* v){ return v->EvaluateString() -> c_str(); } @@ -36,7 +36,7 @@ extern "C" { return new BooleanEvalValue(b); } - EvalValue* CreateStringEvalValue(const char* s){ + EvalValue* CreateStringEvalValue(const char16_t * s){ return new StringEvalValue(s); } } @@ -47,11 +47,12 @@ extern "C" { TEST_CASE( "Evaluate String", "[integration]" ) { - auto script = Script::Create("\"foo bar\""); + auto script = Script::Create(u"\"foo bar\""); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); auto lastValue = script->GetLastValue(); - REQUIRE(std::strcmp(EvaluateEvalValueString(lastValue), "foo bar") == 0); + auto s = u16string(EvaluateEvalValueString(lastValue)); + REQUIRE(s == u"foo bar"); delete script; } diff --git a/src/Evaluator/EvalValues/EvalValue.hpp b/src/Evaluator/EvalValues/EvalValue.hpp index 959383f..9dd373c 100644 --- a/src/Evaluator/EvalValues/EvalValue.hpp +++ b/src/Evaluator/EvalValues/EvalValue.hpp @@ -31,7 +31,7 @@ public: virtual bool EvaluateBool() const{ throw EvaluationException("Can't evaluate this EvalValue as bool."); } - virtual const std::string* EvaluateString() const { + virtual const std::u16string* EvaluateString() const { throw EvaluationException("Can't evaluate this EvalValue as string."); } diff --git a/src/Evaluator/EvalValues/StringEvalValue.hpp b/src/Evaluator/EvalValues/StringEvalValue.hpp index a37c80b..27f8898 100644 --- a/src/Evaluator/EvalValues/StringEvalValue.hpp +++ b/src/Evaluator/EvalValues/StringEvalValue.hpp @@ -9,10 +9,10 @@ using namespace std; class StringEvalValue : public EvalValue{ - string _value; + u16string _value; size_t _hash; public: - explicit StringEvalValue(string s){ + explicit StringEvalValue(u16string s){ _value = move(s); _hash = HashedString::ConstHash (_value.c_str()); } @@ -27,7 +27,7 @@ public: return this->_hash == b->GetHashCode(); }; - const string* EvaluateString() const final{ + const u16string* EvaluateString() const final{ return &_value; } @@ -38,7 +38,7 @@ public: shared_ptr IndexValue(EvalValue* val) final{ // Porygon is 1-indexed, so we convert to that. auto l = val->EvaluateInteger() - 1; - return make_shared(string(1, _value[l])); + return make_shared(u16string(1, _value[l])); } std::size_t GetHashCode() final{ diff --git a/src/Parser/Lexer.cpp b/src/Parser/Lexer.cpp index 5099304..6bec920 100644 --- a/src/Parser/Lexer.cpp +++ b/src/Parser/Lexer.cpp @@ -5,7 +5,7 @@ #include "Lexer.hpp" -Lexer::Lexer(const string& scriptString, class Script* script) +Lexer::Lexer(const u16string& scriptString, class Script* script) : _scriptString(scriptString) { this->_scriptSize = scriptString.size(); @@ -29,19 +29,19 @@ vector Lexer::Lex() { return tokens; } -char Lexer::Peek(){ +char16_t Lexer::Peek(){ if (Lexer::_position >= this -> _scriptSize) return '\0'; return this -> _scriptString.at(Lexer::_position); } -char Lexer::Next(){ - char next = Peek(); +char16_t Lexer::Next(){ + char16_t next = Peek(); Lexer::_position++; return next; } -IToken* Lexer::LexNext(char c){ +IToken* Lexer::LexNext(char16_t c){ switch (c) { case '\0': return new SimpleToken(TokenKind::EndOfFile, this -> _position - 1, 1); @@ -113,7 +113,7 @@ IToken* Lexer::LexNext(char c){ } } -int CharToInt(char c){ +int CharToInt(char16_t c){ switch (c){ case '0': return 0; case '1': return 1; @@ -129,7 +129,7 @@ int CharToInt(char c){ } } -IToken* Lexer::LexNumber(char c){ +IToken* Lexer::LexNumber(char16_t c){ long int_value = CharToInt(c); double float_value = 0; short decimal_index = 0; @@ -138,7 +138,7 @@ IToken* Lexer::LexNumber(char c){ unsigned int start = this -> _position - 1; unsigned int length = 1; while (is_searching){ - char next = this -> Peek(); + char16_t next = this -> Peek(); int next_val = CharToInt(next); if (next_val == -1){ switch (next){ @@ -183,7 +183,7 @@ IToken * Lexer::LexIdentifierOrKeyword() { auto start = this -> _position - 1; auto end = start; while (true){ - char next = this -> Peek(); + char16_t next = this -> Peek(); if (next == '\0') break; if (isalpha(next) || next == '_'){ this -> Next(); @@ -194,7 +194,7 @@ IToken * Lexer::LexIdentifierOrKeyword() { } } - string s = this -> _scriptString.substr(start, end - start + 1); + u16string s = this -> _scriptString.substr(start, end - start + 1); switch (HashedString::ConstHash(s.c_str())){ case HashedString::ConstHash("and"): return new SimpleToken(TokenKind::AndKeyword, start, 3); case HashedString::ConstHash("break"): return new SimpleToken(TokenKind::BreakKeyword, start, 5); @@ -219,7 +219,7 @@ IToken * Lexer::LexIdentifierOrKeyword() { } } -const unordered_map ControlCharacters{ // NOLINT(cert-err58-cpp) +const unordered_map ControlCharacters{ // NOLINT(cert-err58-cpp) {'0', '\0'}, {'a', '\a'}, {'b', '\b'}, @@ -234,12 +234,12 @@ const unordered_map ControlCharacters{ // NOLINT(cert-err58-cpp) {'\\', '\\'}, }; -IToken* Lexer::LexString(char c){ +IToken* Lexer::LexString(char16_t c){ auto start = this -> _position - 1; auto end = start; - char last = c; + char16_t last = c; while (true){ - char next = this -> Peek(); + char16_t next = this -> Peek(); if (next == '\0') break; if (next == c && last != '\\') break; this -> Next(); @@ -252,8 +252,8 @@ IToken* Lexer::LexString(char c){ return new SimpleToken(TokenKind::BadToken, start, end -start + 1); } - string s = this -> _scriptString.substr(start + 1, end - start); - stringstream stream; + u16string s = this -> _scriptString.substr(start + 1, end - start); + std::basic_ostringstream stream; for (int i = 0; i < s.size(); i++){ c = s[i]; if (c == '\\'){ diff --git a/src/Parser/Lexer.hpp b/src/Parser/Lexer.hpp index b6be91a..366d256 100644 --- a/src/Parser/Lexer.hpp +++ b/src/Parser/Lexer.hpp @@ -8,23 +8,23 @@ using namespace std; class Lexer { - const string& _scriptString; + const u16string& _scriptString; #ifdef TESTS_BUILD public: #endif unsigned int _position; unsigned int _scriptSize; - char Peek(); - char Next(); - IToken* LexNext(char c); - IToken* LexNumber(char c); + char16_t Peek(); + char16_t Next(); + IToken* LexNext(char16_t c); + IToken* LexNumber(char16_t c); IToken* LexIdentifierOrKeyword(); - IToken* LexString(char c); + IToken* LexString(char16_t c); public: Script* ScriptData; vector Lex(); - explicit Lexer(const string& scriptString, class Script* script); + explicit Lexer(const u16string& scriptString, class Script* script); }; diff --git a/src/Parser/ParsedExpressions/ParsedExpression.hpp b/src/Parser/ParsedExpressions/ParsedExpression.hpp index 551c235..ae343d7 100644 --- a/src/Parser/ParsedExpressions/ParsedExpression.hpp +++ b/src/Parser/ParsedExpressions/ParsedExpression.hpp @@ -100,7 +100,7 @@ public: }; class LiteralStringExpression : public ParsedExpression{ - const string _value; + const u16string _value; public: const ParsedExpressionKind GetKind() const final{ return ParsedExpressionKind::LiteralString; @@ -111,7 +111,7 @@ public: { } - const string& GetValue() const{ + const u16string& GetValue() const{ return _value; } }; diff --git a/src/Parser/Token.hpp b/src/Parser/Token.hpp index 23d70c4..366e756 100644 --- a/src/Parser/Token.hpp +++ b/src/Parser/Token.hpp @@ -91,10 +91,10 @@ public: }; class StringToken : public IToken{ - const string _value; + const u16string _value; public: - explicit StringToken(string value, unsigned int position, unsigned int length) + explicit StringToken(u16string value, unsigned int position, unsigned int length) : IToken(position, length), _value(std::move(value)) { @@ -104,7 +104,7 @@ public: return TokenKind::String; } - const string& GetValue() const{ + const u16string& GetValue() const{ return _value; } }; diff --git a/src/Script.cpp b/src/Script.cpp index c64f1ee..d42b349 100644 --- a/src/Script.cpp +++ b/src/Script.cpp @@ -1,18 +1,29 @@ #include #include #include +#include #include +#include #include "Script.hpp" #include "Parser/Lexer.hpp" #include "Parser/Parser.hpp" #include "Binder/Binder.hpp" -Script* Script::Create(const string& script) { +Script* Script::Create(const u16string& script) { auto s = new Script(); s -> Parse(script); return s; } +std::u16string To_UTF16(const string &s) +{ + std::wstring_convert, char16_t> conv; + return conv.from_bytes(s); +} +Script *Script::Create(const string &script) { + return Script::Create(To_UTF16(script)); +} + Script::Script() { Diagnostics = new DiagnosticsHolder(); _evaluator = new Evaluator(this); @@ -32,7 +43,7 @@ Script::~Script() { delete this->_scriptVariables; } -void Script::Parse(const string& script) { +void Script::Parse(const u16string& script) { auto lexer = Lexer(script, this); auto lexResult = lexer.Lex(); auto parser = Parser(lexResult, this); @@ -54,11 +65,11 @@ void Script::Parse(const string& script) { delete parseResult; } -EvalValue *Script::GetVariable(const string &key) { +EvalValue *Script::GetVariable(const u16string &key) { return _scriptVariables -> at(HashedString(key).GetHash()).get(); } -bool Script::HasVariable(const string &key) { +bool Script::HasVariable(const u16string &key) { auto f = _scriptVariables->find(HashedString(key).GetHash()); return f != _scriptVariables->end(); } @@ -67,18 +78,19 @@ EvalValue *Script::GetLastValue() { return _evaluator->GetLastValue(); } -bool Script::HasFunction(const string &key) { +bool Script::HasFunction(const u16string &key) { auto f = _scriptVariables->find(HashedString(key).GetHash()); return f != _scriptVariables->end() && f.operator->()->second->GetTypeClass() == TypeClass ::Function; } -shared_ptr Script::CallFunction(const string &key, const vector& variables) { +shared_ptr Script::CallFunction(const u16string &key, const vector& variables) { auto var = (ScriptFunctionEvalValue*)GetVariable(key); return this->_evaluator->EvaluateFunction(var, variables); } + extern "C" { - Script* CreateScript(char * s){ + Script* CreateScript(char16_t * s){ return Script::Create(s); } @@ -90,19 +102,19 @@ extern "C" { return script->GetLastValue(); } - bool HasVariable(Script* script, const char* key){ + bool HasVariable(Script* script, const char16_t* key){ return script->HasVariable(key); } - EvalValue* GetVariable(Script* script, const char* key){ + EvalValue* GetVariable(Script* script, const char16_t* key){ return script->GetVariable(key); } - bool HasFunction(Script* script, const char* key){ + bool HasFunction(Script* script, const char16_t* key){ return script->HasFunction(key); } - EvalValue* CallFunction(Script* script, const char* key, EvalValue* parameters[], int parameterCount){ + EvalValue* CallFunction(Script* script, const char16_t* key, EvalValue* parameters[], int parameterCount){ std::vector v(parameters, parameters + parameterCount); return script->CallFunction(key, v).get(); } diff --git a/src/Script.hpp b/src/Script.hpp index 9b51c97..a95ccf0 100644 --- a/src/Script.hpp +++ b/src/Script.hpp @@ -23,8 +23,9 @@ class Script { shared_ptr _returnType; explicit Script(); - void Parse(const string& script); + void Parse(const u16string& script); public: + static Script* Create(const u16string& script); static Script* Create(const string& script); DiagnosticsHolder* Diagnostics; @@ -42,11 +43,11 @@ public: EvalValue* GetLastValue(); - EvalValue* GetVariable(const string& key); - bool HasVariable(const string& key); + EvalValue* GetVariable(const u16string& key); + bool HasVariable(const u16string& key); - shared_ptr CallFunction(const string& key, const vector& variables); - bool HasFunction(const string& key); + shared_ptr CallFunction(const u16string& key, const vector& variables); + bool HasFunction(const u16string& key); }; diff --git a/src/Utilities/HashedString.hpp b/src/Utilities/HashedString.hpp index bec8973..56d64ce 100644 --- a/src/Utilities/HashedString.hpp +++ b/src/Utilities/HashedString.hpp @@ -7,12 +7,22 @@ class HashedString{ const uint32_t _hash; public: - explicit HashedString(const std::string& s) : _hash(ConstHash(s.c_str())){ + explicit HashedString(const std::u16string& s) : _hash(ConstHash(s.c_str())){ } + explicit HashedString(char16_t const *input) : _hash(ConstHash(input)){ + } + explicit HashedString(char const *input) : _hash(ConstHash(input)){ } + HashedString(const HashedString& b) = default; + static uint32_t constexpr ConstHash(char16_t const *input) { + return *input ? + static_cast(*input) + 33 * ConstHash(input + 1) : + 5381; + } + static uint32_t constexpr ConstHash(char const *input) { return *input ? static_cast(*input) + 33 * ConstHash(input + 1) : diff --git a/tests/integration/ConditionalTests.cpp b/tests/integration/ConditionalTests.cpp index de6309b..10348c2 100644 --- a/tests/integration/ConditionalTests.cpp +++ b/tests/integration/ConditionalTests.cpp @@ -5,10 +5,10 @@ TEST_CASE( "Basic conditional", "[integration]" ) { Script* script = Script::Create("if true then foo = true end"); REQUIRE(!script->Diagnostics -> HasErrors()); - auto variable = script->GetVariable("foo"); + auto variable = script->GetVariable(u"foo"); REQUIRE(variable == nullptr); script->Evaluate(); - variable = script->GetVariable("foo"); + variable = script->GetVariable(u"foo"); REQUIRE(variable != nullptr); REQUIRE(variable->EvaluateBool()); delete script; @@ -17,10 +17,10 @@ TEST_CASE( "Basic conditional", "[integration]" ) { TEST_CASE( "If then, else", "[integration]" ) { Script* script = Script::Create("if false then foo = false else foo = true end"); REQUIRE(!script->Diagnostics -> HasErrors()); - auto variable = script->GetVariable("foo"); + auto variable = script->GetVariable(u"foo"); REQUIRE(variable == nullptr); script->Evaluate(); - variable = script->GetVariable("foo"); + variable = script->GetVariable(u"foo"); REQUIRE(variable != nullptr); REQUIRE(variable->EvaluateBool()); delete script; @@ -29,10 +29,10 @@ TEST_CASE( "If then, else", "[integration]" ) { TEST_CASE( "If then, else if", "[integration]" ) { Script* script = Script::Create("if false then foo = false elseif true then foo = true end"); REQUIRE(!script->Diagnostics -> HasErrors()); - auto variable = script->GetVariable("foo"); + auto variable = script->GetVariable(u"foo"); REQUIRE(variable == nullptr); script->Evaluate(); - variable = script->GetVariable("foo"); + variable = script->GetVariable(u"foo"); REQUIRE(variable != nullptr); REQUIRE(variable->EvaluateBool()); delete script; diff --git a/tests/integration/Functions.cpp b/tests/integration/Functions.cpp index 76a7800..12b0415 100644 --- a/tests/integration/Functions.cpp +++ b/tests/integration/Functions.cpp @@ -6,7 +6,7 @@ TEST_CASE( "Define script function", "[integration]" ) { Script* script = Script::Create("function add(number a, number b) a + b end"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("add"); + auto variable = script->GetVariable(u"add"); REQUIRE(variable != nullptr); REQUIRE(variable->GetTypeClass() == TypeClass::Function); delete script; @@ -16,10 +16,10 @@ TEST_CASE( "Define script function and call", "[integration]" ) { Script* script = Script::Create("function add(number a, number b) result = a + b end add(1, 2)"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("add"); + auto variable = script->GetVariable(u"add"); REQUIRE(variable != nullptr); REQUIRE(variable->GetTypeClass() == TypeClass::Function); - auto result = script->GetVariable("result"); + auto result = script->GetVariable(u"result"); REQUIRE(result->GetTypeClass() == TypeClass::Number); REQUIRE(result->EvaluateInteger() == 3); delete script; @@ -29,10 +29,10 @@ TEST_CASE( "Define script function and call multiple times", "[integration]" ) { Script* script = Script::Create("result = 0 function add(number a) result = result + a end add(1) add(4)"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("add"); + auto variable = script->GetVariable(u"add"); REQUIRE(variable != nullptr); REQUIRE(variable->GetTypeClass() == TypeClass::Function); - auto result = script->GetVariable("result"); + auto result = script->GetVariable(u"result"); REQUIRE(result->GetTypeClass() == TypeClass::Number); REQUIRE(result->EvaluateInteger() == 5); delete script; @@ -43,15 +43,15 @@ TEST_CASE( "Define script function and call from extern", "[integration]" ) { REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - REQUIRE(script->HasFunction("add")); + REQUIRE(script->HasFunction(u"add")); auto toAddVal = new IntegerEvalValue(5); - script->CallFunction("add", {toAddVal}); + script->CallFunction(u"add", {toAddVal}); delete toAddVal; toAddVal = new IntegerEvalValue(6); - script->CallFunction("add", {toAddVal}); + script->CallFunction(u"add", {toAddVal}); delete toAddVal; - auto result = script->GetVariable("result"); + auto result = script->GetVariable(u"result"); REQUIRE(result->GetTypeClass() == TypeClass::Number); REQUIRE(result->EvaluateInteger() == 11); delete script; @@ -67,17 +67,17 @@ TEST_CASE( "Define script function and return", "[integration]" ) { REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - REQUIRE(script->HasFunction("add")); + REQUIRE(script->HasFunction(u"add")); auto toAddVal = new IntegerEvalValue(5); auto toAddVal2 = new IntegerEvalValue(6); - auto result = script->CallFunction("add", {toAddVal, toAddVal2}); + auto result = script->CallFunction(u"add", {toAddVal, toAddVal2}); delete toAddVal; delete toAddVal2; REQUIRE(result->GetTypeClass() == TypeClass::Number); REQUIRE(result->EvaluateInteger() == 11); - auto variable = script->GetVariable("val"); + auto variable = script->GetVariable(u"val"); REQUIRE(variable->GetTypeClass() == TypeClass::Number); REQUIRE(variable->EvaluateInteger() == 0); @@ -97,9 +97,9 @@ end )"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - script->CallFunction("add", {}); + script->CallFunction(u"add", {}); - auto variable = script->GetVariable("val"); + auto variable = script->GetVariable(u"val"); REQUIRE(variable->GetTypeClass() == TypeClass::Number); REQUIRE(variable->EvaluateInteger() == 5); @@ -127,7 +127,7 @@ test() REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("result"); + auto variable = script->GetVariable(u"result"); REQUIRE(variable->GetTypeClass() == TypeClass::Number); REQUIRE(variable->EvaluateInteger() == 50); diff --git a/tests/integration/IndexTests.cpp b/tests/integration/IndexTests.cpp index 6fb1592..d027968 100644 --- a/tests/integration/IndexTests.cpp +++ b/tests/integration/IndexTests.cpp @@ -7,7 +7,7 @@ TEST_CASE( "String indexing", "[integration]" ) { REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); auto lastValue = script->GetLastValue(); - REQUIRE(*lastValue->EvaluateString() == "b"); + REQUIRE(*lastValue->EvaluateString() == u"b"); delete script; } diff --git a/tests/integration/StringOperationsTests.cpp b/tests/integration/StringOperationsTests.cpp index 41428f2..8cb3e3d 100644 --- a/tests/integration/StringOperationsTests.cpp +++ b/tests/integration/StringOperationsTests.cpp @@ -9,7 +9,7 @@ TEST_CASE( "Simple String", "[integration]" ) { REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); auto lastValue = script->GetLastValue(); - REQUIRE(*lastValue->EvaluateString() == "foo bar"); + REQUIRE(*lastValue->EvaluateString() == u"foo bar"); delete script; } @@ -18,7 +18,7 @@ TEST_CASE( "String Concat", "[integration]" ) { REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); auto lastValue = script->GetLastValue(); - REQUIRE(*lastValue->EvaluateString() == "foobar"); + REQUIRE(*lastValue->EvaluateString() == u"foobar"); delete script; } @@ -27,7 +27,7 @@ TEST_CASE( "String Concat 2", "[integration]" ) { REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); auto lastValue = script->GetLastValue(); - REQUIRE(*lastValue->EvaluateString() == "foobar"); + REQUIRE(*lastValue->EvaluateString() == u"foobar"); delete script; } diff --git a/tests/integration/Tables.cpp b/tests/integration/Tables.cpp index 44fcd92..afabd5c 100644 --- a/tests/integration/Tables.cpp +++ b/tests/integration/Tables.cpp @@ -7,7 +7,7 @@ TEST_CASE( "Create empty table", "[integration]" ) { Script* script = Script::Create("table = {}"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("table"); + auto variable = script->GetVariable(u"table"); REQUIRE(variable != nullptr); delete script; } @@ -16,7 +16,7 @@ TEST_CASE( "Create simple integer table", "[integration]" ) { Script* script = Script::Create("table = {100, 50, 20, 5, -100, 50+50}"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("table"); + auto variable = script->GetVariable(u"table"); REQUIRE(variable != nullptr); delete script; } @@ -25,7 +25,7 @@ TEST_CASE( "Create simple string table", "[integration]" ) { Script* script = Script::Create("table = {'bla', 'test', 'foo', 'bar'}"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("table"); + auto variable = script->GetVariable(u"table"); REQUIRE(variable != nullptr); delete script; } @@ -39,7 +39,7 @@ return table[3] REQUIRE(!script->Diagnostics -> HasErrors()); auto variable = script->Evaluate(); REQUIRE(variable != nullptr); - REQUIRE(*variable->EvaluateString() == "foo"); + REQUIRE(*variable->EvaluateString() == u"foo"); delete script; } @@ -53,10 +53,10 @@ table = { )"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("table"); + auto variable = script->GetVariable(u"table"); REQUIRE(variable != nullptr); auto table = (TableEvalValue*)variable; - CHECK(*table->IndexValue("foo")->EvaluateString() == "test"); + CHECK(*table->IndexValue("foo")->EvaluateString() == u"test"); CHECK(table->IndexValue("bar")->EvaluateInteger() == 100); delete script; } diff --git a/tests/integration/UserData.cpp b/tests/integration/UserData.cpp index 58495aa..282f69b 100644 --- a/tests/integration/UserData.cpp +++ b/tests/integration/UserData.cpp @@ -38,7 +38,7 @@ end REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); auto parameter = new UserDataValue(HashedString::ConstHash("testObject"), new UserDataTestObject()); - auto variable = script->CallFunction("testFunc", {parameter}); + auto variable = script->CallFunction(u"testFunc", {parameter}); REQUIRE(variable != nullptr); REQUIRE(variable->EvaluateInteger() == 10); delete script; @@ -55,7 +55,7 @@ end script->Evaluate(); auto obj = new UserDataTestObject(); auto parameter = new UserDataValue(HashedString::ConstHash("testObject"), obj); - script->CallFunction("testFunc", {parameter}); + script->CallFunction(u"testFunc", {parameter}); delete script; REQUIRE(obj->foo == 5000); delete obj; diff --git a/tests/integration/Variables.cpp b/tests/integration/Variables.cpp index 0161f39..be3e172 100644 --- a/tests/integration/Variables.cpp +++ b/tests/integration/Variables.cpp @@ -5,10 +5,10 @@ TEST_CASE( "Create script variable", "[integration]" ) { Script* script = Script::Create("foo = true"); REQUIRE(!script->Diagnostics -> HasErrors()); - auto variable = script->GetVariable("foo"); + auto variable = script->GetVariable(u"foo"); REQUIRE(variable == nullptr); script->Evaluate(); - variable = script->GetVariable("foo"); + variable = script->GetVariable(u"foo"); REQUIRE(variable != nullptr); REQUIRE(variable->EvaluateBool()); delete script; @@ -17,7 +17,7 @@ TEST_CASE( "Create script variable", "[integration]" ) { TEST_CASE( "Create local variable", "[integration]" ) { Script* script = Script::Create("local foo = true"); REQUIRE(!script->Diagnostics -> HasErrors()); - REQUIRE_FALSE(script->HasVariable("foo")); + REQUIRE_FALSE(script->HasVariable(u"foo")); script->Evaluate(); delete script; } @@ -27,7 +27,7 @@ TEST_CASE( "Create script variable and use", "[integration]" ) { "bar = not foo"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("bar"); + auto variable = script->GetVariable(u"bar"); REQUIRE(variable != nullptr); CHECK(variable->EvaluateBool()); delete script; @@ -38,7 +38,7 @@ TEST_CASE( "Create local variable and use", "[integration]" ) { "bar = not foo"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - auto variable = script->GetVariable("bar"); + auto variable = script->GetVariable(u"bar"); REQUIRE(variable != nullptr); CHECK(variable->EvaluateBool()); delete script; @@ -55,9 +55,9 @@ end )"); REQUIRE(!script->Diagnostics -> HasErrors()); script->Evaluate(); - script -> CallFunction("bar", {}); - script -> CallFunction("bar", {}); - auto variable = script->GetVariable("result"); + script -> CallFunction(u"bar", {}); + script -> CallFunction(u"bar", {}); + auto variable = script->GetVariable(u"result"); REQUIRE(variable != nullptr); CHECK(variable->EvaluateInteger() == 2); delete script; diff --git a/tests/parser/LexerTests.cpp b/tests/parser/LexerTests.cpp index 1591b7c..be2e291 100644 --- a/tests/parser/LexerTests.cpp +++ b/tests/parser/LexerTests.cpp @@ -1,14 +1,16 @@ #ifdef TESTS_BUILD #include +#include +#include #include "../../src/Parser/Lexer.hpp" TEST_CASE( "When at end of script return terminator", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); REQUIRE(lexer.Peek() == '\0'); } TEST_CASE( "Peek doesn't advance", "[lexer]" ) { - auto script = new string("5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds + auto script = new u16string(u"5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds Lexer lexer = Lexer(*script, nullptr); REQUIRE(lexer.Peek() == '5'); REQUIRE(lexer.Peek() == '5'); @@ -16,7 +18,7 @@ TEST_CASE( "Peek doesn't advance", "[lexer]" ) { } TEST_CASE( "Next does advance", "[lexer]" ) { - auto script = new string("5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds + auto script = new u16string(u"5 + 5"); // Create as reference to ensure the compiler plays nice with it in release builds Lexer lexer = Lexer(*script, nullptr); REQUIRE(lexer.Next() == '5'); REQUIRE(lexer.Next() == ' '); @@ -27,37 +29,37 @@ TEST_CASE( "Next does advance", "[lexer]" ) { } TEST_CASE( "Lex Null Terminator as EOF", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); REQUIRE(lexer.LexNext('\0') -> GetKind() == TokenKind::EndOfFile); } TEST_CASE( "Lex Plus Token", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); REQUIRE(lexer.LexNext('+') -> GetKind() == TokenKind::PlusToken); } TEST_CASE( "Lex Minus Token", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); REQUIRE(lexer.LexNext('-') -> GetKind() == TokenKind::MinusToken); } TEST_CASE( "Lex Slash Token", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); REQUIRE(lexer.LexNext('/') -> GetKind() == TokenKind::SlashToken); } TEST_CASE( "Lex Star Token", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); REQUIRE(lexer.LexNext('*') -> GetKind() == TokenKind::StarToken); } TEST_CASE( "Lex Assignment Token", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); REQUIRE(lexer.LexNext('=') -> GetKind() == TokenKind::AssignmentToken); } TEST_CASE( "Lex Equality Token", "[lexer]" ) { - Lexer lexer = Lexer("==", nullptr); + Lexer lexer = Lexer(u"==", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; @@ -65,7 +67,7 @@ TEST_CASE( "Lex Equality Token", "[lexer]" ) { } TEST_CASE( "Lex Whitespace", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); CHECK(lexer.LexNext(' ') -> GetKind() == TokenKind::WhiteSpace); CHECK(lexer.LexNext('\t') -> GetKind() == TokenKind::WhiteSpace); CHECK(lexer.LexNext('\n') -> GetKind() == TokenKind::WhiteSpace); @@ -75,7 +77,7 @@ TEST_CASE( "Lex Whitespace", "[lexer]" ) { } TEST_CASE( "Lex Basic Digits", "[lexer]" ) { - Lexer lexer = Lexer("", nullptr); + Lexer lexer = Lexer(u"", nullptr); CHECK(lexer.LexNext('0') -> GetKind() == TokenKind::Integer); CHECK(lexer.LexNext('1') -> GetKind() == TokenKind::Integer); CHECK(lexer.LexNext('2') -> GetKind() == TokenKind::Integer); @@ -88,10 +90,16 @@ TEST_CASE( "Lex Basic Digits", "[lexer]" ) { CHECK(lexer.LexNext('9') -> GetKind() == TokenKind::Integer); } +std::u16string to_u16string(long const &i) { + std::wstring_convert, char16_t> conv; + return conv.from_bytes(std::to_string(i)); +} + TEST_CASE( "Lex Longer Integers", "[lexer]" ) { long integers[] {0,1,5,9,10,50,100,1000,99999,6484,62163,48862}; for (long integer : integers){ - Lexer lexer = Lexer(std::to_string(integer), nullptr); + auto s = to_u16string(integer); + Lexer lexer = Lexer(s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; @@ -101,10 +109,15 @@ TEST_CASE( "Lex Longer Integers", "[lexer]" ) { } } +std::u16string to_u16string(double const &i) { + std::wstring_convert, char16_t> conv; + return conv.from_bytes(std::to_string(i));} + TEST_CASE( "Lex Floats", "[lexer]" ) { double floats[] {0.5, 0.8, 100.7, 52.3548, 8461354.1324886}; for (double f : floats){ - Lexer lexer = Lexer(std::to_string(f), nullptr); + auto s = to_u16string(f); + Lexer lexer = Lexer(s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; @@ -115,133 +128,134 @@ TEST_CASE( "Lex Floats", "[lexer]" ) { } TEST_CASE( "Lex And Keyword", "[lexer]" ) { - Lexer lexer = Lexer("and", nullptr); + Lexer lexer = Lexer(u"and", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::AndKeyword); } TEST_CASE( "Lex Break Keyword", "[lexer]" ) { - Lexer lexer = Lexer("break", nullptr); + Lexer lexer = Lexer(u"break", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::BreakKeyword); } TEST_CASE( "Lex Do Keyword", "[lexer]" ) { - Lexer lexer = Lexer("do", nullptr); + Lexer lexer = Lexer(u"do", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::DoKeyword); } TEST_CASE( "Lex else Keyword", "[lexer]" ) { - Lexer lexer = Lexer("else", nullptr); + Lexer lexer = Lexer(u"else", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::ElseKeyword); } TEST_CASE( "Lex else if Keyword", "[lexer]" ) { - Lexer lexer = Lexer("elseif", nullptr); + Lexer lexer = Lexer(u"elseif", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::ElseIfKeyword); } TEST_CASE( "Lex end Keyword", "[lexer]" ) { - Lexer lexer = Lexer("end", nullptr); + Lexer lexer = Lexer(u"end", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::EndKeyword); } TEST_CASE( "Lex false Keyword", "[lexer]" ) { - Lexer lexer = Lexer("false", nullptr); + Lexer lexer = Lexer(u"false", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::FalseKeyword); } TEST_CASE( "Lex for Keyword", "[lexer]" ) { - Lexer lexer = Lexer("for", nullptr); + Lexer lexer = Lexer(u"for", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::ForKeyword); } TEST_CASE( "Lex function Keyword", "[lexer]" ) { - Lexer lexer = Lexer("function", nullptr); + auto s = new u16string(u"function"); + Lexer lexer = Lexer(*s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::FunctionKeyword); } TEST_CASE( "Lex if Keyword", "[lexer]" ) { - Lexer lexer = Lexer("if", nullptr); + Lexer lexer = Lexer(u"if", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::IfKeyword); } TEST_CASE( "Lex in Keyword", "[lexer]" ) { - Lexer lexer = Lexer("in", nullptr); + Lexer lexer = Lexer(u"in", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::InKeyword); } TEST_CASE( "Lex local Keyword", "[lexer]" ) { - Lexer lexer = Lexer("local", nullptr); + Lexer lexer = Lexer(u"local", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::LocalKeyword); } TEST_CASE( "Lex nil Keyword", "[lexer]" ) { - Lexer lexer = Lexer("nil", nullptr); + Lexer lexer = Lexer(u"nil", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::NilKeyword); } TEST_CASE( "Lex not Keyword", "[lexer]" ) { - Lexer lexer = Lexer("not", nullptr); + Lexer lexer = Lexer(u"not", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::NotKeyword); } TEST_CASE( "Lex or Keyword", "[lexer]" ) { - Lexer lexer = Lexer("or", nullptr); + Lexer lexer = Lexer(u"or", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::OrKeyword); } TEST_CASE( "Lex return Keyword", "[lexer]" ) { - Lexer lexer = Lexer("return", nullptr); + Lexer lexer = Lexer(u"return", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::ReturnKeyword); } TEST_CASE( "Lex then Keyword", "[lexer]" ) { - Lexer lexer = Lexer("then", nullptr); + Lexer lexer = Lexer(u"then", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::ThenKeyword); } TEST_CASE( "Lex true Keyword", "[lexer]" ) { - Lexer lexer = Lexer("true", nullptr); + Lexer lexer = Lexer(u"true", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::TrueKeyword); } TEST_CASE( "Lex while Keyword", "[lexer]" ) { - Lexer lexer = Lexer("while", nullptr); + Lexer lexer = Lexer(u"while", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; @@ -249,7 +263,7 @@ TEST_CASE( "Lex while Keyword", "[lexer]" ) { } TEST_CASE( "Lex identifier", "[lexer]" ) { - Lexer lexer = Lexer("foo", nullptr); + Lexer lexer = Lexer(u"foo", nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; @@ -258,7 +272,8 @@ TEST_CASE( "Lex identifier", "[lexer]" ) { } TEST_CASE( "Lex Start Position", "[lexer]" ) { - Lexer lexer = Lexer("+ - bar 1234", nullptr); + auto s = new u16string(u"+ - bar 1234"); + Lexer lexer = Lexer(*s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 5); CHECK(((IdentifierToken*)tokens[0]) -> GetStartPosition() == 0); @@ -269,7 +284,8 @@ TEST_CASE( "Lex Start Position", "[lexer]" ) { } TEST_CASE( "Lex End Position", "[lexer]" ) { - Lexer lexer = Lexer("+ - bar 1234", nullptr); + auto s = new u16string(u"+ - bar 1234"); + Lexer lexer = Lexer(*s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 5); CHECK(((IdentifierToken*)tokens[0]) -> GetEndPosition() == 0); @@ -280,39 +296,43 @@ TEST_CASE( "Lex End Position", "[lexer]" ) { } TEST_CASE("Lex Double Quote String", "[lexer]") { - Lexer lexer = Lexer("\"foo bar\"", nullptr); + auto s = new u16string(u"\"foo bar\""); + Lexer lexer = Lexer(*s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::String); - REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar"); + REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo bar"); } TEST_CASE("Lex Single Quote String", "[lexer]") { - Lexer lexer = Lexer("'foo bar'", nullptr); + auto s = new u16string(u"'foo bar'"); + Lexer lexer = Lexer(*s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::String); - REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo bar"); + REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo bar"); } TEST_CASE("Lex Double Quote String, Escape Quote", "[lexer]") { - Lexer lexer = Lexer("'foo\\\"bar'", nullptr); + auto s = new u16string(u"'foo\\\"bar'"); + Lexer lexer = Lexer(*s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::String); - REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\"bar"); + REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo\"bar"); } TEST_CASE("Lex String with newline", "[lexer]") { - Lexer lexer = Lexer("'foo\\nbar'", nullptr); + auto s = new u16string(u"'foo\\nbar'"); + Lexer lexer = Lexer(*s, nullptr); auto tokens = lexer.Lex(); REQUIRE(tokens.size() == 2); const IToken* firstToken = tokens[0]; REQUIRE(firstToken -> GetKind() == TokenKind::String); - REQUIRE(((StringToken*)firstToken) -> GetValue() == "foo\nbar"); + REQUIRE(((StringToken*)firstToken) -> GetValue() == u"foo\nbar"); } diff --git a/tests/parser/ParserTests.cpp b/tests/parser/ParserTests.cpp index ff374ad..ba5b3b5 100644 --- a/tests/parser/ParserTests.cpp +++ b/tests/parser/ParserTests.cpp @@ -142,7 +142,7 @@ TEST_CASE( "Assert binary precedence", "[parser]" ) { } TEST_CASE( "Parse String Tokens", "[parser]" ) { - vector v {new StringToken("foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)}; + vector v {new StringToken(u"foo bar", 0,0), new SimpleToken(TokenKind::EndOfFile,0,0)}; Parser parser = Parser(v, nullptr); auto parsedStatements = parser.Parse() -> GetStatements(); REQUIRE(parsedStatements->size() == 1); @@ -151,7 +151,7 @@ TEST_CASE( "Parse String Tokens", "[parser]" ) { auto expression = ((ParsedExpressionStatement*)firstStatement)->GetExpression(); REQUIRE(expression -> GetKind() == ParsedExpressionKind::LiteralString); auto boolean = ((LiteralStringExpression*)expression); - REQUIRE(boolean->GetValue() == "foo bar"); + REQUIRE(boolean->GetValue() == u"foo bar"); } TEST_CASE( "Parse Global Assignment", "[parser]" ) {