Move Lexer to u16string handling, for unicode support

src/Script.cpp

@@ -1,18 +1,29 @@
 #include <utility>
 #include <vector>
 #include <iterator>
+#include <locale>
 #include <unordered_map>
+#include <codecvt>
 #include "Script.hpp"
 #include "Parser/Lexer.hpp"
 #include "Parser/Parser.hpp"
 #include "Binder/Binder.hpp"
 
-Script* Script::Create(const string& script) {
+Script* Script::Create(const u16string& script) {
     auto s = new Script();
     s -> Parse(script);
     return s;
 }
 
+std::u16string To_UTF16(const string &s)
+{
+    std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> conv;
+    return conv.from_bytes(s);
+}
+Script *Script::Create(const string &script) {
+    return Script::Create(To_UTF16(script));
+}
+
 Script::Script() {
     Diagnostics = new DiagnosticsHolder();
     _evaluator = new Evaluator(this);
@@ -32,7 +43,7 @@ Script::~Script() {
     delete this->_scriptVariables;
 }
 
-void Script::Parse(const string& script) {
+void Script::Parse(const u16string& script) {
     auto lexer = Lexer(script, this);
     auto lexResult = lexer.Lex();
     auto parser = Parser(lexResult, this);
@@ -54,11 +65,11 @@ void Script::Parse(const string& script) {
     delete parseResult;
 }
 
-EvalValue *Script::GetVariable(const string &key) {
+EvalValue *Script::GetVariable(const u16string &key) {
     return _scriptVariables -> at(HashedString(key).GetHash()).get();
 }
 
-bool Script::HasVariable(const string &key) {
+bool Script::HasVariable(const u16string &key) {
     auto f = _scriptVariables->find(HashedString(key).GetHash());
     return f != _scriptVariables->end();
 }
@@ -67,18 +78,19 @@ EvalValue *Script::GetLastValue() {
     return _evaluator->GetLastValue();
 }
 
-bool Script::HasFunction(const string &key) {
+bool Script::HasFunction(const u16string &key) {
     auto f = _scriptVariables->find(HashedString(key).GetHash());
     return f != _scriptVariables->end() && f.operator->()->second->GetTypeClass() == TypeClass ::Function;
 }
 
-shared_ptr<EvalValue> Script::CallFunction(const string &key, const vector<EvalValue *>& variables) {
+shared_ptr<EvalValue> Script::CallFunction(const u16string &key, const vector<EvalValue *>& variables) {
     auto var = (ScriptFunctionEvalValue*)GetVariable(key);
     return this->_evaluator->EvaluateFunction(var, variables);
 }
 
+
 extern "C" {
-    Script* CreateScript(char * s){
+    Script* CreateScript(char16_t * s){
         return Script::Create(s);
     }
 
@@ -90,19 +102,19 @@ extern "C" {
         return script->GetLastValue();
     }
 
-    bool HasVariable(Script* script, const char* key){
+    bool HasVariable(Script* script, const char16_t* key){
         return script->HasVariable(key);
     }
 
-    EvalValue* GetVariable(Script* script, const char* key){
+    EvalValue* GetVariable(Script* script, const char16_t* key){
         return script->GetVariable(key);
     }
 
-    bool HasFunction(Script* script, const char* key){
+    bool HasFunction(Script* script, const char16_t* key){
         return script->HasFunction(key);
     }
 
-    EvalValue* CallFunction(Script* script, const char* key, EvalValue* parameters[], int parameterCount){
+    EvalValue* CallFunction(Script* script, const char16_t* key, EvalValue* parameters[], int parameterCount){
         std::vector<EvalValue*> v(parameters, parameters + parameterCount);
         return script->CallFunction(key, v).get();
     }