Rework of ScriptResolver to binary handling. Now also serialises the type database to the stream, simplifying it's api.

src/ScriptResolving/AngelScript/AngelScriptResolver.cpp

@@ -210,43 +210,93 @@ void AngelScriptResolver::CreateScript(const char* name, const char* script) {
 }
 void AngelScriptResolver::WriteByteCodeToFile(const char* file, bool stripDebugInfo) {
     FILE* wFile = nullptr;
+    // Open file in write binary mode.
     wFile = fopen(file, "wb");
+    // Ensure we opened it.
     AssertNotNull(wFile);
-    auto stream = new FileByteCodeStream(wFile);
+    // Save the initial position, as we need to write here later.
+    fpos_t startPos;
+    fgetpos(wFile, &startPos);
+    // The first 8 bytes are the position of the data for which script name is which script decl.
+    // We don't know this position yet, so we write an empty value here. We write a size of 8, to accommodate an
+    // unsigned long.
+    uint64_t byteCodeSizeArr[]{0};
+    fwrite(byteCodeSizeArr, sizeof(uint64_t), 1, wFile);
+    // We initialize a stream, with no bounds (as this is a read thing)
+    auto stream = new FileByteCodeStream(wFile, SIZE_MAX);
+    // And save the angelscript byte code.
     _mainModule->SaveByteCode(stream, stripDebugInfo);
+    // We grab the current position of the written file. This is the position the types will be written on. So we need
+    // to know this.
+    uint64_t bytecodeSize = (uint64_t)ftell(wFile);
+    stream->WriteTypes(_typeDatabase);
+
+    // Go back to the start of the file
+    fsetpos(wFile, &startPos);
+    // And write the actual position the types are at.
+    byteCodeSizeArr[0] = bytecodeSize;
+    fwrite(byteCodeSizeArr, sizeof(uint64_t), 1, wFile);
+
+    // Close the file
     Assert(fclose(wFile) == 0);
     delete stream;
 }
-void AngelScriptResolver::LoadByteCodeFromFile(
+void AngelScriptResolver::LoadByteCodeFromFile(const char* file) {
-    const char* file, const Dictionary<ScriptCategory, Dictionary<ConstString, const char*>>& types) {
     FILE* rFile = nullptr;
+    // Open the file in read binary mode
     rFile = fopen(file, "rb");
+    // Ensure it opened
     AssertNotNull(rFile);
-    auto stream = new FileByteCodeStream(rFile);
+    // the first 8 bytes are position of the types database, everything before that is the angelscript byte code.
+    uint64_t byteCodeSize[]{0};
+    fread(byteCodeSize, sizeof(uint64_t), 1, rFile);
+    // Initialize a stream, with the earlier found bounds.
+    auto stream = new FileByteCodeStream(rFile, byteCodeSize[0] - 1);
+    // And load the angelscript byte code.
+    int result = _mainModule->LoadByteCode(stream);
+    // Ensure we succeeded in this.
+    Assert(result == asSUCCESS);
+
+    // Begin loading the type database
+    auto types = stream->ReadTypes();
+
     InitializeByteCode(stream, types);
     Assert(fclose(rFile) == 0);
     delete stream;
 }
 uint8_t* AngelScriptResolver::WriteByteCodeToMemory(size_t& size, bool stripDebugInfo) {
     auto stream = new MemoryByteCodeStream();
+    size_t byteCodeSize[]{0};
+    stream->Write(byteCodeSize, sizeof(uint64_t));
     auto result = _mainModule->SaveByteCode(stream, stripDebugInfo);
     Assert(result == asSUCCESS);
+    byteCodeSize[0] = (uint64_t)stream->GetWrittenSize();
+    stream->WriteTypes(_typeDatabase);
+    stream->WriteToPosition(byteCodeSize, sizeof(uint64_t), 0);
     auto arr = stream->GetOut();
     size = stream->GetWrittenSize();
     arr = static_cast<uint8_t*>(realloc(arr, size * sizeof(uint8_t)));
     delete stream;
     return arr;
 }
-void AngelScriptResolver::LoadByteCodeFromMemory(
+void AngelScriptResolver::LoadByteCodeFromMemory(uint8_t* byte, size_t size) {
-    uint8_t* byte, size_t size, const Dictionary<ScriptCategory, Dictionary<ConstString, const char*>>& types) {
     auto stream = new MemoryByteCodeStream(byte, size);
+    uint64_t byteCodeSizeArr[]{0};
+    stream->Read(byteCodeSizeArr, sizeof(uint64_t));
+    stream->SetAngelScriptBound((size_t)byteCodeSizeArr[0]);
+    // And load the angelscript byte code.
+    int result = _mainModule->LoadByteCode(stream);
+    // Ensure we succeeded in this.
+    Assert(result == asSUCCESS);
+
+    // Begin loading the type database
+    auto types = stream->ReadTypes();
+
     InitializeByteCode(stream, types);
     delete stream;
 }
 void AngelScriptResolver::InitializeByteCode(
-    asIBinaryStream* stream, const Dictionary<ScriptCategory, Dictionary<ConstString, const char*>>& types) {
+    asIBinaryStream* stream, const Dictionary<ScriptCategory, Dictionary<ConstString, uint32_t>>& types) {
-    int result = _mainModule->LoadByteCode(stream);
-    Assert(result == asSUCCESS);
 
     auto typeCount = _mainModule->GetObjectTypeCount();
     Dictionary<uint32_t, asITypeInfo*> objectTypes;
@@ -255,11 +305,11 @@ void AngelScriptResolver::InitializeByteCode(
         objectTypes.Insert(ConstString::GetHash(t->GetName()), t);
     }
     Dictionary<ScriptCategory, Dictionary<ConstString, AngelScriptTypeInfo*>> typeDatabase;
-    for (auto& innerDb : types) {
+    for (const auto& innerDb : types) {
         Dictionary<ConstString, AngelScriptTypeInfo*> newInnerDb;
-        for (auto& val : innerDb.second) {
+        for (const auto& val : innerDb.second) {
             auto decl = val.second;
-            auto type = objectTypes[ConstString::GetHash(decl)];
+            auto type = objectTypes[decl];
             newInnerDb.Insert(val.first, new AngelScriptTypeInfo(val.first, type));
         }
         typeDatabase.Insert(innerDb.first, newInnerDb);

src/ScriptResolving/AngelScript/AngelScriptResolver.hpp

@@ -26,7 +26,7 @@ private:
 
     void RegisterTypes();
     void InitializeByteCode(asIBinaryStream* stream,
-                            const Dictionary<ScriptCategory, Dictionary<ConstString, const char*>>& types);
+                            const Dictionary<ScriptCategory, Dictionary<ConstString, uint32_t>>& types);
 
 public:
     ~AngelScriptResolver() override {
@@ -47,11 +47,9 @@ public:
     CreatureLib::Battling::Script* LoadScript(ScriptCategory category, const ConstString& scriptName) override;
 
     void WriteByteCodeToFile(const char* file, bool stripDebugInfo = false);
-    void LoadByteCodeFromFile(const char* file,
+    void LoadByteCodeFromFile(const char* file);
-                              const Dictionary<ScriptCategory, Dictionary<ConstString, const char*>>& types);
     uint8_t* WriteByteCodeToMemory(size_t& size, bool stripDebugInfo = false);
-    void LoadByteCodeFromMemory(uint8_t*, size_t size,
+    void LoadByteCodeFromMemory(uint8_t*, size_t size);
-                                const Dictionary<ScriptCategory, Dictionary<ConstString, const char*>>& types);
 
     const Dictionary<ScriptCategory, Dictionary<ConstString, AngelScriptTypeInfo*>>& GetTypeDatabase() const noexcept {
         return _typeDatabase;

src/ScriptResolving/AngelScript/ByteCodeHandling/FileByteCodeStream.hpp

@@ -2,23 +2,30 @@
 #define PKMNLIB_FILEBYTECODESTREAM_HPP
 #include <angelscript.h>
 #include <cstdio>
+#include "IPkmnBinaryStream.hpp"
 
-class FileByteCodeStream : public asIBinaryStream {
+class FileByteCodeStream : public IPkmnBinaryStream {
 private:
     FILE* _file;
+    size_t _readPosition = 0;
 
 public:
-    explicit FileByteCodeStream(FILE* file) : _file(file) {}
+    explicit FileByteCodeStream(FILE* file, size_t bound) : IPkmnBinaryStream(bound), _file(file) {}
 
-    int Write(const void* ptr, asUINT size) {
+    int Write(const void* ptr, asUINT size) override {
         if (size == 0)
             return 0;
         return fwrite(ptr, size, 1, _file);
     }
-    int Read(void* ptr, asUINT size) {
+    int Read(void* ptr, asUINT size) override {
         if (size == 0)
             return 0;
-        return fread(ptr, size, 1, _file);
+        if (_readPosition + size >= _angelScriptBound) {
+            size = _angelScriptBound - _readPosition;
+        }
+        auto diff = fread(ptr, size, 1, _file);
+        _readPosition += diff;
+        return diff;
     }
 };
 

src/ScriptResolving/AngelScript/ByteCodeHandling/IPkmnBinaryStream.hpp

@@ -0,0 +1,111 @@
+#ifndef PKMNLIB_IPKMNBINARYSTREAM_HPP
+#define PKMNLIB_IPKMNBINARYSTREAM_HPP
+
+#include <Arbutils/Collections/Dictionary.hpp>
+#include <Arbutils/ConstString.hpp>
+#include <CreatureLib/Battling/ScriptHandling/ScriptCategory.hpp>
+#include <angelscript.h>
+
+class IPkmnBinaryStream : public asIBinaryStream {
+protected:
+    size_t _angelScriptBound;
+
+    IPkmnBinaryStream(size_t angelScriptBound) : _angelScriptBound(angelScriptBound) {}
+
+public:
+    virtual void WriteTypes(const Dictionary<ScriptCategory, Dictionary<ConstString, AngelScriptTypeInfo*>>& types) {
+        // We serialize our types in the format
+        // "[category(byte)][name(str)]\2[decl(str)]\2[name(str)]\2[decl(str)]\1[category(byte)]...."
+
+        ScriptCategory categoryArr[1];
+        for (const auto& dic : types) {
+            // Write the category
+            categoryArr[0] = dic.first;
+            Write(categoryArr, sizeof(ScriptCategory));
+            for (const auto& inner : dic.second) {
+                // Write the script name
+                Write(inner.first.c_str(), sizeof(char) * inner.first.Length());
+                // Write the divider
+                Write("\2", sizeof(char));
+                // Write the declaration of the script
+                auto decl = inner.second->GetDecl();
+                Write(decl, sizeof(char) * strlen(decl));
+                // Write another divider.
+                Write("\2", sizeof(char));
+            }
+            // Write the divider between categories.
+            Write("\1", sizeof(char));
+        }
+    }
+    virtual Dictionary<ScriptCategory, Dictionary<ConstString, uint32_t>> ReadTypes() {
+        _angelScriptBound = SIZE_MAX;
+        Dictionary<ScriptCategory, Dictionary<ConstString, uint32_t>> types;
+        ScriptCategory categoryArr[1];
+        while (true) {
+            // Every inner database starts with the category, of known size. Read that.
+            auto read = Read(categoryArr, sizeof(ScriptCategory));
+            // If we haven't read anything, we are finished.
+            if (read == 0) {
+                break;
+            }
+            Dictionary<ConstString, uint32_t> innerDb;
+
+            // We don't know the sizes of the name and decl. Allocate 128 characters for them, as that should be enough.
+            char name[128];
+            char decl[128];
+            size_t pos = 0;
+            bool isDecl = false;
+
+            while (true) {
+                // Keep reading characters
+                char cArr[1];
+                Read(cArr, sizeof(char));
+                auto c = cArr[0];
+                // If we find a '\1' separator
+                if (c == '\1') {
+                    // and if we were reading the decl
+                    if (isDecl) {
+                        // Insert the name and decl into the dictionary. Close off the decl with eof as well.
+                        decl[pos] = '\0';
+                        innerDb.Insert(Arbutils::CaseInsensitiveConstString(name),
+                                       Arbutils::CaseInsensitiveConstString::GetHash(decl));
+                    }
+                    // If we have found \1, we are done with the current category, so break.
+                    break;
+                }
+                // If we find a '\2' separator, we need to toggle between writing to name and writing to decl. If we
+                // were writing a decl, also insert the name and decl into the dictionary.
+                if (c == '\2') {
+                    if (isDecl) {
+                        // Insert the name and decl into the dictionary. Close off the decl with eof as well.
+                        decl[pos] = '\0';
+                        innerDb.Insert(Arbutils::CaseInsensitiveConstString(name),
+                                       Arbutils::CaseInsensitiveConstString::GetHash(decl));
+                        // Reset our position and toggle back to name.
+                        pos = 0;
+                        isDecl = false;
+                        continue;
+                    } else {
+                        // Close of the name with eof, reset position and toggle to decl.
+                        name[pos] = '\0';
+                        pos = 0;
+                        isDecl = true;
+                        continue;
+                    }
+                } else {
+                    // If we haven't found any control character, just add the character to the thing we are writing to,
+                    // and increment the position.
+                    if (isDecl) {
+                        decl[pos++] = c;
+                    } else {
+                        name[pos++] = c;
+                    }
+                }
+            }
+            types.Insert(categoryArr[0], innerDb);
+        }
+        return types;
+    }
+};
+
+#endif // PKMNLIB_IPKMNBINARYSTREAM_HPP

src/ScriptResolving/AngelScript/ByteCodeHandling/MemoryByteCodeStream.hpp

@@ -3,7 +3,7 @@
 #include <angelscript.h>
 #include <vector>
 
-class MemoryByteCodeStream : public asIBinaryStream {
+class MemoryByteCodeStream : public IPkmnBinaryStream {
 private:
     uint8_t* _out;
     size_t _index = 0;
@@ -13,12 +13,15 @@ private:
 #define MEM_STEPS 256
 
 public:
-    MemoryByteCodeStream() : _out((uint8_t*)malloc(MEM_STEPS * sizeof(uint8_t))), _capacity(MEM_STEPS){};
+    MemoryByteCodeStream()
-    MemoryByteCodeStream(uint8_t* in, size_t size) : _out(in), _size(size) {}
+        : IPkmnBinaryStream(SIZE_MAX), _out((uint8_t*)malloc(MEM_STEPS * sizeof(uint8_t))), _capacity(MEM_STEPS){};
+    MemoryByteCodeStream(uint8_t* in, size_t size) : IPkmnBinaryStream(SIZE_MAX), _out(in), _size(size) {}
 
     uint8_t* GetOut() const { return _out; }
     size_t GetWrittenSize() const { return _size; }
 
+    void SetAngelScriptBound(size_t bound) noexcept { _angelScriptBound = bound; }
+
     int Write(const void* ptr, asUINT size) final {
         if (size == 0)
             return 0;
@@ -38,6 +41,14 @@ public:
         _size += size;
         return size;
     }
+
+    void WriteToPosition(const void* ptr, asUINT size, size_t position) {
+        auto start = reinterpret_cast<const uint8_t*>(ptr);
+        for (asUINT index = 0; index < size; index++) {
+            _out[position + index] = *(start + index);
+        }
+    }
+
     int Read(void* ptr, asUINT size) final {
         if (size == 0)
             return 0;

tests/ScriptTests/ScriptResolverTests.cpp

@@ -85,21 +85,10 @@ TEST_CASE("Get a script resolver, save the byte code to memory, create new scrip
     originLib->FinalizeModule();
     size_t size;
     auto byteCode = originLib->WriteByteCodeToMemory(size);
-    auto typeDatabase = originLib->GetTypeDatabase();
-
-    Dictionary<ScriptCategory, Dictionary<ConstString, const char*>> types;
-    for (auto& innerDb : typeDatabase) {
-        Dictionary<ConstString, const char*> newInnerDb;
-        for (auto& kv : innerDb.second) {
-            INFO(kv.second->GetDecl());
-            newInnerDb.Insert(kv.first, kv.second->GetDecl());
-        }
-        types.Insert(innerDb.first, newInnerDb);
-    }
 
     auto newLib = dynamic_cast<AngelScriptResolver*>(PkmnLib::Battling::BattleLibrary::CreateScriptResolver());
     newLib->Initialize(TestLibrary::GetLibrary());
-    newLib->LoadByteCodeFromMemory(byteCode, size, types);
+    newLib->LoadByteCodeFromMemory(byteCode, size);
     auto obj = dynamic_cast<AngelScriptScript*>(newLib->LoadScript(ScriptCategory::Creature, "testScript1"_cnc));
     REQUIRE(obj != nullptr);
     delete obj;
@@ -107,4 +96,22 @@ TEST_CASE("Get a script resolver, save the byte code to memory, create new scrip
     delete newLib;
     free(byteCode);
 }
+
+TEST_CASE("Get a script resolver, save the byte code to file, create new script resolver from it") {
+    auto originLib = dynamic_cast<AngelScriptResolver*>(PkmnLib::Battling::BattleLibrary::CreateScriptResolver());
+    originLib->Initialize(TestLibrary::GetLibrary());
+    originLib->CreateScript("testScript1", _scripts["testScript1"]);
+    originLib->FinalizeModule();
+    originLib->WriteByteCodeToFile("foo.bin");
+    auto newLib = dynamic_cast<AngelScriptResolver*>(PkmnLib::Battling::BattleLibrary::CreateScriptResolver());
+    newLib->Initialize(TestLibrary::GetLibrary());
+    newLib->LoadByteCodeFromFile("foo.bin");
+    auto obj = dynamic_cast<AngelScriptScript*>(newLib->LoadScript(ScriptCategory::Creature, "testScript1"_cnc));
+    REQUIRE(obj != nullptr);
+    delete obj;
+    delete originLib;
+    delete newLib;
+    // remove("foo.bin");
+}
+
 #endif