Rework identifier handling, adds typedef statement.

This commit is contained in:
Deukhoofd 2020-10-08 19:53:02 +02:00
parent e99b1af78d
commit 2fb1b68ead
Signed by: Deukhoofd
GPG Key ID: F63E044490819F6F
9 changed files with 295 additions and 119 deletions

View File

@ -0,0 +1,38 @@
#ifndef MALACHSCRIPT_IDENTIFIER_HPP
#define MALACHSCRIPT_IDENTIFIER_HPP
#include <cstdint>
#include <string_view>
namespace MalachScript {
class Identifier {
std::u8string_view _str;
uint32_t _hash;
public:
Identifier() : _hash(0) {}
consteval Identifier(const char8_t* c) : _str(c), _hash(Hash(c)) {}
Identifier(const std::u8string_view& c) : _str(c), _hash(HashStringView(c)) {}
Identifier(const std::u8string_view& c, uint32_t hash) : _str(c), _hash(hash) {}
[[nodiscard]] constexpr inline uint32_t GetHash() const noexcept { return _hash; }
[[nodiscard]] constexpr inline const std::u8string_view& GetString() const noexcept { return _str; }
constexpr inline static uint32_t Hash(const char8_t* input) {
if (*input != 0U) {
return static_cast<uint32_t>(*input) + 33 * Hash(input + 1);
} else {
return 5381;
}
};
constexpr inline static uint32_t HashStringView(const std::u8string_view& sv) {
auto init = 5381;
for (auto it = sv.rbegin(); it != sv.rend(); ++it) {
init *= 33;
init += static_cast<uint32_t>(*it);
}
return init;
}
};
}
#endif // MALACHSCRIPT_IDENTIFIER_HPP

View File

@ -0,0 +1,42 @@
#ifndef MALACHSCRIPT_PRIMITIVETYPES_HPP
#define MALACHSCRIPT_PRIMITIVETYPES_HPP
#include <string>
#include "Identifier.hpp"
namespace MalachScript {
static constinit Identifier _voidName = u8"void";
static constinit Identifier _intName = u8"int";
static constinit Identifier _int8Name = u8"int8";
static constinit Identifier _int16Name = u8"int16";
static constinit Identifier _int32Name = u8"int32";
static constinit Identifier _int64Name = u8"int64";
static constinit Identifier _uintName = u8"uint";
static constinit Identifier _uint8Name = u8"uint8";
static constinit Identifier _uint16Name = u8"uint16";
static constinit Identifier _uint32Name = u8"uint32";
static constinit Identifier _uint64Name = u8"uint64";
static constinit Identifier _floatName = u8"float";
static constinit Identifier _doubleName = u8"double";
static constinit Identifier _boolName = u8"bool";
class PrimitiveTypes {
public:
constexpr static const Identifier& VoidName() noexcept { return _voidName; }
constexpr static const Identifier& IntName() noexcept { return _intName; }
constexpr static const Identifier& Int8Name() noexcept { return _int8Name; }
constexpr static const Identifier& Int16Name() noexcept { return _int16Name; }
constexpr static const Identifier& Int32Name() noexcept { return _int32Name; }
constexpr static const Identifier& Int64Name() noexcept { return _int64Name; }
constexpr static const Identifier& UintName() noexcept { return _uintName; }
constexpr static const Identifier& Uint8Name() noexcept { return _uint8Name; }
constexpr static const Identifier& Uint16Name() noexcept { return _uint16Name; }
constexpr static const Identifier& Uint32Name() noexcept { return _uint32Name; }
constexpr static const Identifier& Uint64Name() noexcept { return _uint64Name; }
constexpr static const Identifier& FloatName() noexcept { return _floatName; }
constexpr static const Identifier& DoubleName() noexcept { return _doubleName; }
constexpr static const Identifier& BoolName() noexcept { return _boolName; }
};
}
#endif // MALACHSCRIPT_PRIMITIVETYPES_HPP

View File

@ -2,8 +2,9 @@
#define MALACHSCRIPT_LEXTOKEN_HPP #define MALACHSCRIPT_LEXTOKEN_HPP
#include <memory> #include <memory>
#include "LexTokenKind.hpp" #include "../../CoreData/Identifier.hpp"
#include "../../TextSpan.hpp" #include "../../TextSpan.hpp"
#include "LexTokenKind.hpp"
namespace MalachScript::Parser { namespace MalachScript::Parser {
class LexToken { class LexToken {
@ -55,12 +56,12 @@ namespace MalachScript::Parser {
}; };
class IdentifierToken : public LexTokenImpl<LexTokenKind::Identifier> { class IdentifierToken : public LexTokenImpl<LexTokenKind::Identifier> {
std::u8string_view _value; Identifier _value;
public: public:
IdentifierToken(TextSpan span, std::u8string_view value) IdentifierToken(TextSpan span, Identifier value)
: LexTokenImpl<LexTokenKind::Identifier>(span), _value(value) {} : LexTokenImpl<LexTokenKind::Identifier>(span), _value(value) {}
[[nodiscard]] const std::u8string_view& GetValue() const noexcept { return _value; } [[nodiscard]] const Identifier& GetValue() const noexcept { return _value; }
}; };
} }

View File

@ -458,23 +458,6 @@ namespace MalachScript::Parser {
return Create<StringLiteral>(TextSpan(start, start + _position), std::u8string(_script.substr(start, offset))); return Create<StringLiteral>(TextSpan(start, start + _position), std::u8string(_script.substr(start, offset)));
} }
static uint32_t constexpr Hash(const char8_t* input) {
if (*input != 0U) {
return static_cast<uint32_t>(*input) + 33 * Hash(input + 1);
} else {
return 5381;
}
};
static uint32_t HashStringView(const std::u8string_view& sv){
auto init = 5381;
for (auto it = sv.rbegin(); it != sv.rend(); ++it)
{
init *= 33;
init += static_cast<uint32_t>(*it);
}
return init;
}
LexToken* Lexer::LexKeywordOrIdentifier() { LexToken* Lexer::LexKeywordOrIdentifier() {
auto start = _position; auto start = _position;
auto offset = 0; auto offset = 0;
@ -483,87 +466,138 @@ namespace MalachScript::Parser {
} }
auto str = _script.substr(start, offset); auto str = _script.substr(start, offset);
Progress(offset - 1); Progress(offset - 1);
switch (HashStringView(str)) { auto hash = Identifier::HashStringView(str);
case Hash(u8"and"): return Create<LexTokenImpl<LexTokenKind::AndKeyword>>(TextSpan(start, _position)); switch (hash) {
case Hash(u8"abstract"): case Identifier::Hash(u8"and"):
return Create<LexTokenImpl<LexTokenKind::AndKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"abstract"):
return Create<LexTokenImpl<LexTokenKind::AbstractKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::AbstractKeyword>>(TextSpan(start, _position));
case Hash(u8"auto"): return Create<LexTokenImpl<LexTokenKind::AutoKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"auto"):
case Hash(u8"bool"): return Create<LexTokenImpl<LexTokenKind::BoolKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::AutoKeyword>>(TextSpan(start, _position));
case Hash(u8"break"): return Create<LexTokenImpl<LexTokenKind::BreakKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"bool"):
case Hash(u8"case"): return Create<LexTokenImpl<LexTokenKind::CaseKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::BoolKeyword>>(TextSpan(start, _position));
case Hash(u8"cast"): return Create<LexTokenImpl<LexTokenKind::CastKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"break"):
case Hash(u8"catch"): return Create<LexTokenImpl<LexTokenKind::CatchKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::BreakKeyword>>(TextSpan(start, _position));
case Hash(u8"class"): return Create<LexTokenImpl<LexTokenKind::ClassKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"case"):
case Hash(u8"const"): return Create<LexTokenImpl<LexTokenKind::ConstKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::CaseKeyword>>(TextSpan(start, _position));
case Hash(u8"continue"): case Identifier::Hash(u8"cast"):
return Create<LexTokenImpl<LexTokenKind::CastKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"catch"):
return Create<LexTokenImpl<LexTokenKind::CatchKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"class"):
return Create<LexTokenImpl<LexTokenKind::ClassKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"const"):
return Create<LexTokenImpl<LexTokenKind::ConstKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"continue"):
return Create<LexTokenImpl<LexTokenKind::ContinueKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::ContinueKeyword>>(TextSpan(start, _position));
case Hash(u8"default"): case Identifier::Hash(u8"default"):
return Create<LexTokenImpl<LexTokenKind::DefaultKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::DefaultKeyword>>(TextSpan(start, _position));
case Hash(u8"do"): return Create<LexTokenImpl<LexTokenKind::DoKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"do"):
case Hash(u8"double"): return Create<LexTokenImpl<LexTokenKind::DoubleKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::DoKeyword>>(TextSpan(start, _position));
case Hash(u8"else"): return Create<LexTokenImpl<LexTokenKind::ElseKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"double"):
case Hash(u8"enum"): return Create<LexTokenImpl<LexTokenKind::EnumKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::DoubleKeyword>>(TextSpan(start, _position));
case Hash(u8"explicit"): case Identifier::Hash(u8"else"):
return Create<LexTokenImpl<LexTokenKind::ElseKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"enum"):
return Create<LexTokenImpl<LexTokenKind::EnumKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"explicit"):
return Create<LexTokenImpl<LexTokenKind::ExplicitKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::ExplicitKeyword>>(TextSpan(start, _position));
case Hash(u8"external"): case Identifier::Hash(u8"external"):
return Create<LexTokenImpl<LexTokenKind::ExternalKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::ExternalKeyword>>(TextSpan(start, _position));
case Hash(u8"false"): return Create<LexTokenImpl<LexTokenKind::FalseKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"false"):
case Hash(u8"final"): return Create<LexTokenImpl<LexTokenKind::FinalKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::FalseKeyword>>(TextSpan(start, _position));
case Hash(u8"float"): return Create<LexTokenImpl<LexTokenKind::FloatKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"final"):
case Hash(u8"for"): return Create<LexTokenImpl<LexTokenKind::ForKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::FinalKeyword>>(TextSpan(start, _position));
case Hash(u8"from"): return Create<LexTokenImpl<LexTokenKind::FromKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"float"):
case Hash(u8"funcdef"): return Create<LexTokenImpl<LexTokenKind::FloatKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"for"):
return Create<LexTokenImpl<LexTokenKind::ForKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"from"):
return Create<LexTokenImpl<LexTokenKind::FromKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"funcdef"):
return Create<LexTokenImpl<LexTokenKind::FuncdefKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::FuncdefKeyword>>(TextSpan(start, _position));
case Hash(u8"function"): case Identifier::Hash(u8"function"):
return Create<LexTokenImpl<LexTokenKind::FunctionKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::FunctionKeyword>>(TextSpan(start, _position));
case Hash(u8"get"): return Create<LexTokenImpl<LexTokenKind::GetKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"get"):
case Hash(u8"if"): return Create<LexTokenImpl<LexTokenKind::IfKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::GetKeyword>>(TextSpan(start, _position));
case Hash(u8"import"): return Create<LexTokenImpl<LexTokenKind::ImportKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"if"):
case Hash(u8"in"): return Create<LexTokenImpl<LexTokenKind::InKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::IfKeyword>>(TextSpan(start, _position));
case Hash(u8"inout"): return Create<LexTokenImpl<LexTokenKind::InoutKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"import"):
case Hash(u8"int"): return Create<LexTokenImpl<LexTokenKind::IntKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::ImportKeyword>>(TextSpan(start, _position));
case Hash(u8"interface"): case Identifier::Hash(u8"in"):
return Create<LexTokenImpl<LexTokenKind::InKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"inout"):
return Create<LexTokenImpl<LexTokenKind::InoutKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"int"):
return Create<LexTokenImpl<LexTokenKind::IntKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"interface"):
return Create<LexTokenImpl<LexTokenKind::InterfaceKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::InterfaceKeyword>>(TextSpan(start, _position));
case Hash(u8"int8"): return Create<LexTokenImpl<LexTokenKind::Int8Keyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"int8"):
case Hash(u8"int16"): return Create<LexTokenImpl<LexTokenKind::Int16Keyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::Int8Keyword>>(TextSpan(start, _position));
case Hash(u8"int32"): return Create<LexTokenImpl<LexTokenKind::Int32Keyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"int16"):
case Hash(u8"int64"): return Create<LexTokenImpl<LexTokenKind::Int64Keyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::Int16Keyword>>(TextSpan(start, _position));
case Hash(u8"is"): return Create<LexTokenImpl<LexTokenKind::IsKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"int32"):
case Hash(u8"mixin"): return Create<LexTokenImpl<LexTokenKind::MixinKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::Int32Keyword>>(TextSpan(start, _position));
case Hash(u8"namespace"): case Identifier::Hash(u8"int64"):
return Create<LexTokenImpl<LexTokenKind::Int64Keyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"is"):
return Create<LexTokenImpl<LexTokenKind::IsKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"mixin"):
return Create<LexTokenImpl<LexTokenKind::MixinKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"namespace"):
return Create<LexTokenImpl<LexTokenKind::NamespaceKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::NamespaceKeyword>>(TextSpan(start, _position));
case Hash(u8"not"): return Create<LexTokenImpl<LexTokenKind::NotKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"not"):
case Hash(u8"null"): return Create<LexTokenImpl<LexTokenKind::NullKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::NotKeyword>>(TextSpan(start, _position));
case Hash(u8"or"): return Create<LexTokenImpl<LexTokenKind::OrKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"null"):
case Hash(u8"out"): return Create<LexTokenImpl<LexTokenKind::OutKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::NullKeyword>>(TextSpan(start, _position));
case Hash(u8"override"): case Identifier::Hash(u8"or"):
return Create<LexTokenImpl<LexTokenKind::OrKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"out"):
return Create<LexTokenImpl<LexTokenKind::OutKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"override"):
return Create<LexTokenImpl<LexTokenKind::OverrideKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::OverrideKeyword>>(TextSpan(start, _position));
case Hash(u8"private"): case Identifier::Hash(u8"private"):
return Create<LexTokenImpl<LexTokenKind::PrivateKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::PrivateKeyword>>(TextSpan(start, _position));
case Hash(u8"property"): case Identifier::Hash(u8"property"):
return Create<LexTokenImpl<LexTokenKind::PropertyKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::PropertyKeyword>>(TextSpan(start, _position));
case Hash(u8"protected"): case Identifier::Hash(u8"protected"):
return Create<LexTokenImpl<LexTokenKind::ProtectedKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::ProtectedKeyword>>(TextSpan(start, _position));
case Hash(u8"return"): return Create<LexTokenImpl<LexTokenKind::ReturnKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"return"):
case Hash(u8"set"): return Create<LexTokenImpl<LexTokenKind::SetKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::ReturnKeyword>>(TextSpan(start, _position));
case Hash(u8"shared"): return Create<LexTokenImpl<LexTokenKind::SharedKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"set"):
case Hash(u8"super"): return Create<LexTokenImpl<LexTokenKind::SuperKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::SetKeyword>>(TextSpan(start, _position));
case Hash(u8"switch"): return Create<LexTokenImpl<LexTokenKind::SwitchKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"shared"):
case Hash(u8"this"): return Create<LexTokenImpl<LexTokenKind::ThisKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::SharedKeyword>>(TextSpan(start, _position));
case Hash(u8"true"): return Create<LexTokenImpl<LexTokenKind::TrueKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"super"):
case Hash(u8"try"): return Create<LexTokenImpl<LexTokenKind::TryKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::SuperKeyword>>(TextSpan(start, _position));
case Hash(u8"typedef"): case Identifier::Hash(u8"switch"):
return Create<LexTokenImpl<LexTokenKind::SwitchKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"this"):
return Create<LexTokenImpl<LexTokenKind::ThisKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"true"):
return Create<LexTokenImpl<LexTokenKind::TrueKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"try"):
return Create<LexTokenImpl<LexTokenKind::TryKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"typedef"):
return Create<LexTokenImpl<LexTokenKind::TypedefKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::TypedefKeyword>>(TextSpan(start, _position));
case Hash(u8"uint"): return Create<LexTokenImpl<LexTokenKind::UintKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"uint"):
case Hash(u8"uint8"): return Create<LexTokenImpl<LexTokenKind::Uint8Keyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::UintKeyword>>(TextSpan(start, _position));
case Hash(u8"uint16"): return Create<LexTokenImpl<LexTokenKind::Uint16Keyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"uint8"):
case Hash(u8"uint32"): return Create<LexTokenImpl<LexTokenKind::Uint32Keyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::Uint8Keyword>>(TextSpan(start, _position));
case Hash(u8"uint64"): return Create<LexTokenImpl<LexTokenKind::Uint64Keyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"uint16"):
case Hash(u8"void"): return Create<LexTokenImpl<LexTokenKind::VoidKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::Uint16Keyword>>(TextSpan(start, _position));
case Hash(u8"while"): return Create<LexTokenImpl<LexTokenKind::WhileKeyword>>(TextSpan(start, _position)); case Identifier::Hash(u8"uint32"):
case Hash(u8"xor"): return Create<LexTokenImpl<LexTokenKind::XorKeyword>>(TextSpan(start, _position)); return Create<LexTokenImpl<LexTokenKind::Uint32Keyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"uint64"):
return Create<LexTokenImpl<LexTokenKind::Uint64Keyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"void"):
return Create<LexTokenImpl<LexTokenKind::VoidKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"while"):
return Create<LexTokenImpl<LexTokenKind::WhileKeyword>>(TextSpan(start, _position));
case Identifier::Hash(u8"xor"):
return Create<LexTokenImpl<LexTokenKind::XorKeyword>>(TextSpan(start, _position));
default: return Create<IdentifierToken>(TextSpan(start, _position), str); default: return Create<IdentifierToken>(TextSpan(start, _position), Identifier(str, hash));
} }
} }
bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) { bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) {

View File

@ -1,10 +1,16 @@
#include "Parser.hpp" #include "Parser.hpp"
#include <iostream> #include <iostream>
#include "../CoreData/PrimitiveTypes.hpp"
#define PROGRESS_TOKEN(token) \ #define PROGRESS_TOKEN(token) \
token = token->GetNext().get(); \ token = (token)->GetNext().get(); \
while (token->GetKind() == LexTokenKind::Whitespace) { \ while ((token)->GetKind() == LexTokenKind::Whitespace) { \
token = token->GetNext().get(); \ (token) = (token)->GetNext().get(); \
}
#define EXPECT_TOKEN(token, kind) \
if (token->GetKind() != LexTokenKind::kind) { \
LogError(Diagnostics::DiagnosticType::UnexpectedToken, token->GetSpan()); \
} }
namespace MalachScript::Parser { namespace MalachScript::Parser {
@ -48,11 +54,11 @@ namespace MalachScript::Parser {
} }
PROGRESS_TOKEN(current); PROGRESS_TOKEN(current);
} }
std::u8string_view identifier;
// After class keyword, an identifier should always follow, if it doesn't, log an error. // After class keyword, an identifier should always follow, if it doesn't, log an error.
identifier = ParseIdentifier(current, encounteredError); Identifier identifier;
ParseIdentifier(identifier, current, encounteredError);
PROGRESS_TOKEN(current); PROGRESS_TOKEN(current);
std::vector<std::u8string_view> inherits; std::vector<Identifier> inherits;
std::vector<const ParsedStatement*> body; std::vector<const ParsedStatement*> body;
body.reserve(16); body.reserve(16);
@ -63,16 +69,16 @@ namespace MalachScript::Parser {
} }
case LexTokenKind::ColonSymbol: { case LexTokenKind::ColonSymbol: {
PROGRESS_TOKEN(current); PROGRESS_TOKEN(current);
auto id = ParseIdentifier(current, encounteredError); Identifier id;
ParseIdentifier(id, current, encounteredError);
inherits.push_back(id); inherits.push_back(id);
while (current->GetKind() == LexTokenKind::CommaSymbol) { while (current->GetKind() == LexTokenKind::CommaSymbol) {
PROGRESS_TOKEN(current); PROGRESS_TOKEN(current);
id = ParseIdentifier(current, encounteredError); ParseIdentifier(id, current, encounteredError);
inherits.push_back(id); inherits.push_back(id);
PROGRESS_TOKEN(current); PROGRESS_TOKEN(current);
} }
if (!encounteredError && current->GetKind() != LexTokenKind::OpenCurlyParenthesisSymbol) if (!encounteredError && current->GetKind() != LexTokenKind::OpenCurlyParenthesisSymbol) {
{
encounteredError = true; encounteredError = true;
LogError(Diagnostics::DiagnosticType::UnexpectedToken, current->GetSpan()); LogError(Diagnostics::DiagnosticType::UnexpectedToken, current->GetSpan());
} }
@ -88,12 +94,10 @@ namespace MalachScript::Parser {
} }
const ParsedStatement* statement; const ParsedStatement* statement;
// TODO: Sort by // TODO: Sort by
if (!ParseVirtProp(statement) && if (!ParseVirtProp(statement) && !ParseFunc(statement) && !ParseVar(statement) &&
!ParseFunc(statement) && !ParseFuncDef(statement)) {
!ParseVar(statement) &&
!ParseFuncDef(statement)){
LogError(Diagnostics::DiagnosticType::UnexpectedToken, current->GetSpan()); LogError(Diagnostics::DiagnosticType::UnexpectedToken, current->GetSpan());
} else{ } else {
body.push_back(statement); body.push_back(statement);
} }
} }
@ -101,13 +105,56 @@ namespace MalachScript::Parser {
} }
default: throw; default: throw;
} }
out = new ParsedClassStatement(TextSpan(start, current->GetSpan().GetEnd()), identifier, out = new ParsedClassStatement(TextSpan(start, current->GetSpan().GetEnd()), identifier, inherits, body);
inherits, body);
_currentToken = current; _currentToken = current;
return true; return true;
} }
bool Parser::ParseVirtProp([[maybe_unused]]const ParsedStatement*& out) { return false; } bool Parser::ParseTypeDef(const ParsedStatement*& out) {
bool Parser::ParseFunc([[maybe_unused]]const ParsedStatement*& out) { return false; } if (_currentToken->GetKind() != LexTokenKind::TypedefKeyword) {
bool Parser::ParseVar([[maybe_unused]]const ParsedStatement*& out) { return false; } return false;
bool Parser::ParseFuncDef([[maybe_unused]]const ParsedStatement*& out) { return false; } }
auto start = _currentToken->GetSpan().GetStart();
PROGRESS_TOKEN(_currentToken);
Identifier defineFrom;
auto encounteredErrors = false;
if (!ParsePrimType(defineFrom) && !ParseIdentifier(defineFrom, _currentToken, encounteredErrors)) {
LogError(Diagnostics::DiagnosticType::UnexpectedToken, _currentToken->GetSpan());
}
PROGRESS_TOKEN(_currentToken);
Identifier defineTo;
if (!ParseIdentifier(defineTo, _currentToken, encounteredErrors)) {
LogError(Diagnostics::DiagnosticType::UnexpectedToken, _currentToken->GetSpan());
}
PROGRESS_TOKEN(_currentToken);
if (_currentToken->GetKind() != LexTokenKind::SemicolonSymbol) {
LogError(Diagnostics::DiagnosticType::UnexpectedToken, _currentToken->GetSpan());
}
out = new ParsedTypeDefStatement(TextSpan(start, _currentToken->GetSpan().GetEnd()), defineTo, defineFrom);
return true;
}
bool Parser::ParseVirtProp([[maybe_unused]] const ParsedStatement*& out) { return false; }
bool Parser::ParseFunc([[maybe_unused]] const ParsedStatement*& out) { return false; }
bool Parser::ParseVar([[maybe_unused]] const ParsedStatement*& out) { return false; }
bool Parser::ParseFuncDef([[maybe_unused]] const ParsedStatement*& out) { return false; }
bool Parser::ParsePrimType(Identifier& out) {
// TODO: out needs to return a value.
switch (_currentToken->GetKind()) {
case LexTokenKind::VoidKeyword: out = PrimitiveTypes::VoidName(); return true;
case LexTokenKind::IntKeyword: out = PrimitiveTypes::IntName(); return true;
case LexTokenKind::Int8Keyword: out = PrimitiveTypes::Int8Name(); return true;
case LexTokenKind::Int16Keyword: out = PrimitiveTypes::Int16Name(); return true;
case LexTokenKind::Int32Keyword: out = PrimitiveTypes::Int32Name(); return true;
case LexTokenKind::Int64Keyword: out = PrimitiveTypes::Int64Name(); return true;
case LexTokenKind::UintKeyword: PrimitiveTypes::UintName(); return true;
case LexTokenKind::Uint8Keyword: PrimitiveTypes::Uint8Name(); return true;
case LexTokenKind::Uint16Keyword: PrimitiveTypes::Uint16Name(); return true;
case LexTokenKind::Uint32Keyword: PrimitiveTypes::Uint32Name(); return true;
case LexTokenKind::Uint64Keyword: PrimitiveTypes::Uint64Name(); return true;
case LexTokenKind::FloatKeyword: PrimitiveTypes::FloatName(); return true;
case LexTokenKind::DoubleKeyword: PrimitiveTypes::DoubleName(); return true;
case LexTokenKind::BoolKeyword: PrimitiveTypes::BoolName(); return true;
default: return false;
}
}
} }

View File

@ -21,18 +21,21 @@ namespace MalachScript::Parser {
} }
bool ParseClass(const ParsedStatement*& out); bool ParseClass(const ParsedStatement*& out);
bool ParseTypeDef(const ParsedStatement*& out);
bool ParseVirtProp(const ParsedStatement*& out); bool ParseVirtProp(const ParsedStatement*& out);
bool ParseFunc(const ParsedStatement*& out); bool ParseFunc(const ParsedStatement*& out);
bool ParseVar(const ParsedStatement*& out); bool ParseVar(const ParsedStatement*& out);
bool ParseFuncDef(const ParsedStatement*& out); bool ParseFuncDef(const ParsedStatement*& out);
std::u8string_view ParseIdentifier(const LexToken* token, bool& logError) { bool ParsePrimType(Identifier& out);
bool ParseIdentifier(Identifier& out, const LexToken* token, bool& logError) {
if (logError && token->GetKind() != LexTokenKind::Identifier) { if (logError && token->GetKind() != LexTokenKind::Identifier) {
LogError(Diagnostics::DiagnosticType::UnexpectedToken, token->GetSpan()); LogError(Diagnostics::DiagnosticType::UnexpectedToken, token->GetSpan());
logError = false; logError = false;
return std::u8string_view(); return false;
} }
return reinterpret_cast<const IdentifierToken*>(token)->GetValue(); out = reinterpret_cast<const IdentifierToken*>(token)->GetValue();
return true;
} }
}; };
} }

View File

@ -37,12 +37,12 @@ namespace MalachScript::Parser {
}; };
class ParsedClassStatement : public ParsedStatementImpl<ParsedStatementKind::Class> { class ParsedClassStatement : public ParsedStatementImpl<ParsedStatementKind::Class> {
std::u8string_view _identifier; Identifier _identifier;
std::vector<std::u8string_view> _inherits; std::vector<Identifier> _inherits;
std::vector<std::unique_ptr<const ParsedStatement>> _body; std::vector<std::unique_ptr<const ParsedStatement>> _body;
public: public:
ParsedClassStatement(TextSpan span, std::u8string_view identifier, std::vector<std::u8string_view> inherits, ParsedClassStatement(TextSpan span, Identifier identifier, std::vector<Identifier> inherits,
const std::vector<const ParsedStatement*>& body) const std::vector<const ParsedStatement*>& body)
: ParsedStatementImpl<ParsedStatementKind::Class>(span), _identifier(identifier), _inherits(inherits), : ParsedStatementImpl<ParsedStatementKind::Class>(span), _identifier(identifier), _inherits(inherits),
_body(body.size()) { _body(body.size()) {
@ -50,6 +50,18 @@ namespace MalachScript::Parser {
_body[i] = std::unique_ptr<const ParsedStatement>(body[i]); _body[i] = std::unique_ptr<const ParsedStatement>(body[i]);
} }
}; };
class ParsedTypeDefStatement : public ParsedStatementImpl<ParsedStatementKind::TypeDef> {
Identifier _defineFrom;
Identifier _defineTo;
public:
ParsedTypeDefStatement(TextSpan span, const Identifier& defineFrom, const Identifier& defineTo)
: ParsedStatementImpl<ParsedStatementKind::TypeDef>(span), _defineFrom(defineFrom), _defineTo(defineTo) {}
inline const Identifier& GetDefineFrom() const noexcept { return _defineFrom; }
inline const Identifier& GetDefineTo() const noexcept { return _defineTo; }
};
} }
#endif // MALACHSCRIPT_PARSEDSTATEMENT_HPP #endif // MALACHSCRIPT_PARSEDSTATEMENT_HPP

View File

@ -4,10 +4,9 @@
namespace MalachScript::Parser { namespace MalachScript::Parser {
enum class ParsedStatementKind : uint8_t { enum class ParsedStatementKind : uint8_t {
Unknown, Unknown,
Block,
Script, Script,
ExpressionStatement,
Class, Class,
TypeDef
}; };
} }

View File

@ -97,7 +97,7 @@ namespace doctest {
const auto* token = lexer.Lex(); \ const auto* token = lexer.Lex(); \
CHECK(diag.GetMessages().empty()); \ CHECK(diag.GetMessages().empty()); \
REQUIRE(token->GetKind() == LexTokenKind::Identifier); \ REQUIRE(token->GetKind() == LexTokenKind::Identifier); \
auto value = ((IdentifierToken*)token)->GetValue(); \ auto value = ((IdentifierToken*)token)->GetValue().GetString(); \
CHECK(value == std::u8string(reinterpret_cast<const char8_t*>(identifier))); \ CHECK(value == std::u8string(reinterpret_cast<const char8_t*>(identifier))); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \ CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
} }