Deukhoofd
/
MalachScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lex identifiers and keywords.

This commit is contained in:
Deukhoofd 2020-10-04 18:30:53 +02:00
parent db7ad0bd76
commit 20976010d6
Signed by: Deukhoofd
GPG Key ID: F63E044490819F6F
5 changed files with 352 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/Parser/Lexer/LexToken.hpp
View File

@ -46,6 +46,14 @@ namespace ElohimScript::Parser {
StringLiteral(std::u8string value) : _value(std::move(value)) {} StringLiteral(std::u8string value) : _value(std::move(value)) {}
[[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; } [[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; }
}; };
class IdentifierToken : public LexTokenImpl<LexTokenKind::Identifier>{
std::u8string _value;
public:
IdentifierToken(std::u8string value) : _value(std::move(value)) {}
[[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; }
};
} }
#endif // ELOHIMSCRIPT_LEXTOKEN_HPP #endif // ELOHIMSCRIPT_LEXTOKEN_HPP

67
src/Parser/Lexer/LexTokenKind.hpp
View File

@ -62,10 +62,77 @@ namespace ElohimScript::Parser {
ExclamationMarkIsSymbol, ExclamationMarkIsSymbol,
ColonColonSymbol, ColonColonSymbol,
// Keywords
AndKeyword,
AbstractKeyword,
AutoKeyword,
BoolKeyword,
BreakKeyword,
CaseKeyword,
CastKeyword,
CatchKeyword,
ClassKeyword,
ConstKeyword,
ContinueKeyword,
DefaultKeyword,
DoKeyword,
DoubleKeyword,
ElseKeyword,
EnumKeyword,
ExplicitKeyword,
ExternalKeyword,
FalseKeyword,
FinalKeyword,
FloatKeyword,
ForKeyword,
FromKeyword,
FuncdefKeyword,
FunctionKeyword,
GetKeyword,
IfKeyword,
ImportKeyword,
InKeyword,
InoutKeyword,
IntKeyword,
InterfaceKeyword,
Int8Keyword,
Int16Keyword,
Int32Keyword,
Int64Keyword,
IsKeyword,
MixinKeyword,
NamespaceKeyword,
NotKeyword,
NullKeyword,
OrKeyword,
OutKeyword,
OverrideKeyword,
PrivateKeyword,
PropertyKeyword,
ProtectedKeyword,
ReturnKeyword,
SetKeyword,
SharedKeyword,
SuperKeyword,
SwitchKeyword,
ThisKeyword,
TrueKeyword,
TryKeyword,
TypedefKeyword,
UintKeyword,
Uint8Keyword,
Uint16Keyword,
Uint32Keyword,
Uint64Keyword,
VoidKeyword,
WhileKeyword,
XorKeyword,
// Literals // Literals
FloatLiteral, FloatLiteral,
IntegerLiteral, IntegerLiteral,
StringLiteral, StringLiteral,
Identifier,
}; };
} }

167
src/Parser/Lexer/Lexer.cpp
View File

@ -223,7 +223,11 @@ namespace ElohimScript::Parser {
return LexString(u8'"', false); return LexString(u8'"', false);
} }
default: return new LexTokenImpl<LexTokenKind::Unknown>(); default:
if (IsAlphaNumericalOrUnderscore(c))
return LexKeywordOrIdentifier();
// TODO: Log error
return new LexTokenImpl<LexTokenKind::Unknown>();
} }
} }
@ -399,4 +403,165 @@ namespace ElohimScript::Parser {
} }
return new StringLiteral(std::u8string(_script.substr(start, offset))); return new StringLiteral(std::u8string(_script.substr(start, offset)));
} }
static uint32_t constexpr Hash(const char8_t* input) {
return *input ? static_cast<uint32_t>(*input) + 33 * Hash(input + 1) : 5381;
};
LexToken* Lexer::LexKeywordOrIdentifier() {
auto offset = 0;
while (IsAlphaNumericalOrUnderscore(Peek(offset))) {
offset++;
}
auto str = _script.substr(_position, offset);
Progress(offset);
switch (Hash(str.data())) {
case Hash(u8"and"):
return new LexTokenImpl<LexTokenKind::AndKeyword>();
case Hash(u8"abstract"):
return new LexTokenImpl<LexTokenKind::AbstractKeyword>();
case Hash(u8"auto"):
return new LexTokenImpl<LexTokenKind::AutoKeyword>();
case Hash(u8"bool"):
return new LexTokenImpl<LexTokenKind::BoolKeyword>();
case Hash(u8"break"):
return new LexTokenImpl<LexTokenKind::BreakKeyword>();
case Hash(u8"case"):
return new LexTokenImpl<LexTokenKind::CaseKeyword>();
case Hash(u8"cast"):
return new LexTokenImpl<LexTokenKind::CastKeyword>();
case Hash(u8"catch"):
return new LexTokenImpl<LexTokenKind::CatchKeyword>();
case Hash(u8"class"):
return new LexTokenImpl<LexTokenKind::ClassKeyword>();
case Hash(u8"const"):
return new LexTokenImpl<LexTokenKind::ConstKeyword>();
case Hash(u8"continue"):
return new LexTokenImpl<LexTokenKind::ContinueKeyword>();
case Hash(u8"default"):
return new LexTokenImpl<LexTokenKind::DefaultKeyword>();
case Hash(u8"do"):
return new LexTokenImpl<LexTokenKind::DoKeyword>();
case Hash(u8"double"):
return new LexTokenImpl<LexTokenKind::DoubleKeyword>();
case Hash(u8"else"):
return new LexTokenImpl<LexTokenKind::ElseKeyword>();
case Hash(u8"enum"):
return new LexTokenImpl<LexTokenKind::EnumKeyword>();
case Hash(u8"explicit"):
return new LexTokenImpl<LexTokenKind::ExplicitKeyword>();
case Hash(u8"external"):
return new LexTokenImpl<LexTokenKind::ExternalKeyword>();
case Hash(u8"false"):
return new LexTokenImpl<LexTokenKind::FalseKeyword>();
case Hash(u8"final"):
return new LexTokenImpl<LexTokenKind::FinalKeyword>();
case Hash(u8"float"):
return new LexTokenImpl<LexTokenKind::FloatKeyword>();
case Hash(u8"for"):
return new LexTokenImpl<LexTokenKind::ForKeyword>();
case Hash(u8"from"):
return new LexTokenImpl<LexTokenKind::FromKeyword>();
case Hash(u8"funcdef"):
return new LexTokenImpl<LexTokenKind::FuncdefKeyword>();
case Hash(u8"function"):
return new LexTokenImpl<LexTokenKind::FunctionKeyword>();
case Hash(u8"get"):
return new LexTokenImpl<LexTokenKind::GetKeyword>();
case Hash(u8"if"):
return new LexTokenImpl<LexTokenKind::IfKeyword>();
case Hash(u8"import"):
return new LexTokenImpl<LexTokenKind::ImportKeyword>();
case Hash(u8"in"):
return new LexTokenImpl<LexTokenKind::InKeyword>();
case Hash(u8"inout"):
return new LexTokenImpl<LexTokenKind::InoutKeyword>();
case Hash(u8"int"):
return new LexTokenImpl<LexTokenKind::IntKeyword>();
case Hash(u8"interface"):
return new LexTokenImpl<LexTokenKind::InterfaceKeyword>();
case Hash(u8"int8"):
return new LexTokenImpl<LexTokenKind::Int8Keyword>();
case Hash(u8"int16"):
return new LexTokenImpl<LexTokenKind::Int16Keyword>();
case Hash(u8"int32"):
return new LexTokenImpl<LexTokenKind::Int32Keyword>();
case Hash(u8"int64"):
return new LexTokenImpl<LexTokenKind::Int64Keyword>();
case Hash(u8"is"):
return new LexTokenImpl<LexTokenKind::IsKeyword>();
case Hash(u8"mixin"):
return new LexTokenImpl<LexTokenKind::MixinKeyword>();
case Hash(u8"namespace"):
return new LexTokenImpl<LexTokenKind::NamespaceKeyword>();
case Hash(u8"not"):
return new LexTokenImpl<LexTokenKind::NotKeyword>();
case Hash(u8"null"):
return new LexTokenImpl<LexTokenKind::NullKeyword>();
case Hash(u8"or"):
return new LexTokenImpl<LexTokenKind::OrKeyword>();
case Hash(u8"out"):
return new LexTokenImpl<LexTokenKind::OutKeyword>();
case Hash(u8"override"):
return new LexTokenImpl<LexTokenKind::OverrideKeyword>();
case Hash(u8"private"):
return new LexTokenImpl<LexTokenKind::PrivateKeyword>();
case Hash(u8"property"):
return new LexTokenImpl<LexTokenKind::PropertyKeyword>();
case Hash(u8"protected"):
return new LexTokenImpl<LexTokenKind::ProtectedKeyword>();
case Hash(u8"return"):
return new LexTokenImpl<LexTokenKind::ReturnKeyword>();
case Hash(u8"set"):
return new LexTokenImpl<LexTokenKind::SetKeyword>();
case Hash(u8"shared"):
return new LexTokenImpl<LexTokenKind::SharedKeyword>();
case Hash(u8"super"):
return new LexTokenImpl<LexTokenKind::SuperKeyword>();
case Hash(u8"switch"):
return new LexTokenImpl<LexTokenKind::SwitchKeyword>();
case Hash(u8"this"):
return new LexTokenImpl<LexTokenKind::ThisKeyword>();
case Hash(u8"true"):
return new LexTokenImpl<LexTokenKind::TrueKeyword>();
case Hash(u8"try"):
return new LexTokenImpl<LexTokenKind::TryKeyword>();
case Hash(u8"typedef"):
return new LexTokenImpl<LexTokenKind::TypedefKeyword>();
case Hash(u8"uint"):
return new LexTokenImpl<LexTokenKind::UintKeyword>();
case Hash(u8"uint8"):
return new LexTokenImpl<LexTokenKind::Uint8Keyword>();
case Hash(u8"uint16"):
return new LexTokenImpl<LexTokenKind::Uint16Keyword>();
case Hash(u8"uint32"):
return new LexTokenImpl<LexTokenKind::Uint32Keyword>();
case Hash(u8"uint64"):
return new LexTokenImpl<LexTokenKind::Uint64Keyword>();
case Hash(u8"void"):
return new LexTokenImpl<LexTokenKind::VoidKeyword>();
case Hash(u8"while"):
return new LexTokenImpl<LexTokenKind::WhileKeyword>();
case Hash(u8"xor"):
return new LexTokenImpl<LexTokenKind::XorKeyword>();
default:
return new IdentifierToken(std::u8string(str));
}
}
bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) {
if (c >= 'a' && c <= 'z') {
return true;
}
if (c >= 'A' && c <= 'Z') {
return true;
}
if (c >= '0' && c <= '9') {
return true;
}
if (c == '_') {
return true;
}
return false;
}
} }

3
src/Parser/Lexer/Lexer.hpp
View File

@ -43,6 +43,9 @@ namespace ElohimScript::Parser {
IntegerLiteral* LexBinary(); IntegerLiteral* LexBinary();
StringLiteral* LexString(char8_t opening, bool heredoc); StringLiteral* LexString(char8_t opening, bool heredoc);
LexToken* LexKeywordOrIdentifier();
static bool IsAlphaNumericalOrUnderscore(char8_t c);
}; };
} }

108
tests/LexerTests/IdentifierLexTests.cpp Normal file
View File

@ -0,0 +1,108 @@
#include "../../extern/doctest.hpp"
#include "../../src/Parser/Lexer/Lexer.hpp"
using namespace ElohimScript::Parser;
#define KEYWORD_TEST(script, symbol) \
TEST_CASE("Lex " script) { \
auto lexer = Lexer(script); \
const auto* token = lexer.Lex(); \
CHECK(token->GetKind() == LexTokenKind::symbol); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
KEYWORD_TEST("and", AndKeyword);
KEYWORD_TEST("abstract", AbstractKeyword);
KEYWORD_TEST("auto", AutoKeyword);
KEYWORD_TEST("bool", BoolKeyword);
KEYWORD_TEST("break", BreakKeyword);
KEYWORD_TEST("case", CaseKeyword);
KEYWORD_TEST("cast", CastKeyword);
KEYWORD_TEST("catch", CatchKeyword);
KEYWORD_TEST("class", ClassKeyword);
KEYWORD_TEST("const", ConstKeyword);
KEYWORD_TEST("continue", ContinueKeyword);
KEYWORD_TEST("default", DefaultKeyword);
KEYWORD_TEST("do", DoKeyword);
KEYWORD_TEST("double", DoubleKeyword);
KEYWORD_TEST("else", ElseKeyword);
KEYWORD_TEST("enum", EnumKeyword);
KEYWORD_TEST("explicit", ExplicitKeyword);
KEYWORD_TEST("external", ExternalKeyword);
KEYWORD_TEST("false", FalseKeyword);
KEYWORD_TEST("final", FinalKeyword);
KEYWORD_TEST("float", FloatKeyword);
KEYWORD_TEST("for", ForKeyword);
KEYWORD_TEST("from", FromKeyword);
KEYWORD_TEST("funcdef", FuncdefKeyword);
KEYWORD_TEST("function", FunctionKeyword);
KEYWORD_TEST("get", GetKeyword);
KEYWORD_TEST("if", IfKeyword);
KEYWORD_TEST("import", ImportKeyword);
KEYWORD_TEST("in", InKeyword);
KEYWORD_TEST("inout", InoutKeyword);
KEYWORD_TEST("int", IntKeyword);
KEYWORD_TEST("interface", InterfaceKeyword);
KEYWORD_TEST("int8", Int8Keyword);
KEYWORD_TEST("int16", Int16Keyword);
KEYWORD_TEST("int32", Int32Keyword);
KEYWORD_TEST("int64", Int64Keyword);
KEYWORD_TEST("is", IsKeyword);
KEYWORD_TEST("mixin", MixinKeyword);
KEYWORD_TEST("namespace", NamespaceKeyword);
KEYWORD_TEST("not", NotKeyword);
KEYWORD_TEST("null", NullKeyword);
KEYWORD_TEST("or", OrKeyword);
KEYWORD_TEST("out", OutKeyword);
KEYWORD_TEST("override", OverrideKeyword);
KEYWORD_TEST("private", PrivateKeyword);
KEYWORD_TEST("property", PropertyKeyword);
KEYWORD_TEST("protected", ProtectedKeyword);
KEYWORD_TEST("return", ReturnKeyword);
KEYWORD_TEST("set", SetKeyword);
KEYWORD_TEST("shared", SharedKeyword);
KEYWORD_TEST("super", SuperKeyword);
KEYWORD_TEST("switch", SwitchKeyword);
KEYWORD_TEST("this", ThisKeyword);
KEYWORD_TEST("true", TrueKeyword);
KEYWORD_TEST("try", TryKeyword);
KEYWORD_TEST("typedef", TypedefKeyword);
KEYWORD_TEST("uint", UintKeyword);
KEYWORD_TEST("uint8", Uint8Keyword);
KEYWORD_TEST("uint16", Uint16Keyword);
KEYWORD_TEST("uint32", Uint32Keyword);
KEYWORD_TEST("uint64", Uint64Keyword);
KEYWORD_TEST("void", VoidKeyword);
KEYWORD_TEST("while", WhileKeyword);
KEYWORD_TEST("xor", XorKeyword);
namespace doctest {
template <> struct StringMaker<std::u8string> {
static String convert(const std::u8string& value) {
return String(reinterpret_cast<const char*>(value.data()));
}
};
template <> struct StringMaker<LexTokenKind> {
static String convert(LexTokenKind value) { return String(std::to_string((uint32_t)value).c_str()); }
};
}
#define IDENTIFIER_TEST(identifier) \
TEST_CASE("Lex identifier " identifier) { \
auto lexer = Lexer(identifier); \
const auto* token = lexer.Lex(); \
REQUIRE(token->GetKind() == LexTokenKind::Identifier); \
auto value = ((IdentifierToken*)token)->GetValue(); \
CHECK(value == std::u8string(reinterpret_cast<const char8_t*>(identifier))); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
IDENTIFIER_TEST("foobar");
IDENTIFIER_TEST("_foobar");
IDENTIFIER_TEST("_foo8bar");
#undef KEYWORD_TEST
#undef IDENTIFIER_TEST