Deukhoofd/MalachScript
1
0 Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lex identifiers and keywords.

Browse Source
This commit is contained in:
Deukhoofd committed 2020-10-04 18:30:53 +02:00
1 parent db7ad0bd76
commit 20976010d6
5 files changed
+352 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/Parser/Lexer/LexToken.hpp
+8
View File
@@ -46,6 +46,14 @@ namespace ElohimScript::Parser {
StringLiteral(std::u8string value) : _value(std::move(value)) {}
[[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; }
};
class IdentifierToken : public LexTokenImpl<LexTokenKind::Identifier>{
std::u8string _value;
public:
IdentifierToken(std::u8string value) : _value(std::move(value)) {}
[[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; }
};
}
#endif // ELOHIMSCRIPT_LEXTOKEN_HPP
src/Parser/Lexer/LexTokenKind.hpp
+67
View File
@@ -62,10 +62,77 @@ namespace ElohimScript::Parser {
ExclamationMarkIsSymbol,
ColonColonSymbol,
// Keywords
AndKeyword,
AbstractKeyword,
AutoKeyword,
BoolKeyword,
BreakKeyword,
CaseKeyword,
CastKeyword,
CatchKeyword,
ClassKeyword,
ConstKeyword,
ContinueKeyword,
DefaultKeyword,
DoKeyword,
DoubleKeyword,
ElseKeyword,
EnumKeyword,
ExplicitKeyword,
ExternalKeyword,
FalseKeyword,
FinalKeyword,
FloatKeyword,
ForKeyword,
FromKeyword,
FuncdefKeyword,
FunctionKeyword,
GetKeyword,
IfKeyword,
ImportKeyword,
InKeyword,
InoutKeyword,
IntKeyword,
InterfaceKeyword,
Int8Keyword,
Int16Keyword,
Int32Keyword,
Int64Keyword,
IsKeyword,
MixinKeyword,
NamespaceKeyword,
NotKeyword,
NullKeyword,
OrKeyword,
OutKeyword,
OverrideKeyword,
PrivateKeyword,
PropertyKeyword,
ProtectedKeyword,
ReturnKeyword,
SetKeyword,
SharedKeyword,
SuperKeyword,
SwitchKeyword,
ThisKeyword,
TrueKeyword,
TryKeyword,
TypedefKeyword,
UintKeyword,
Uint8Keyword,
Uint16Keyword,
Uint32Keyword,
Uint64Keyword,
VoidKeyword,
WhileKeyword,
XorKeyword,
// Literals
FloatLiteral,
IntegerLiteral,
StringLiteral,
Identifier,
};
}
src/Parser/Lexer/Lexer.cpp
+166 -1
View File
@@ -223,7 +223,11 @@ namespace ElohimScript::Parser {
return LexString(u8'"', false);
}
default: return new LexTokenImpl<LexTokenKind::Unknown>();
default:
if (IsAlphaNumericalOrUnderscore(c))
return LexKeywordOrIdentifier();
// TODO: Log error
return new LexTokenImpl<LexTokenKind::Unknown>();
}
}
@@ -399,4 +403,165 @@ namespace ElohimScript::Parser {
}
return new StringLiteral(std::u8string(_script.substr(start, offset)));
}
static uint32_t constexpr Hash(const char8_t* input) {
return *input ? static_cast<uint32_t>(*input) + 33 * Hash(input + 1) : 5381;
};
LexToken* Lexer::LexKeywordOrIdentifier() {
auto offset = 0;
while (IsAlphaNumericalOrUnderscore(Peek(offset))) {
offset++;
}
auto str = _script.substr(_position, offset);
Progress(offset);
switch (Hash(str.data())) {
case Hash(u8"and"):
return new LexTokenImpl<LexTokenKind::AndKeyword>();
case Hash(u8"abstract"):
return new LexTokenImpl<LexTokenKind::AbstractKeyword>();
case Hash(u8"auto"):
return new LexTokenImpl<LexTokenKind::AutoKeyword>();
case Hash(u8"bool"):
return new LexTokenImpl<LexTokenKind::BoolKeyword>();
case Hash(u8"break"):
return new LexTokenImpl<LexTokenKind::BreakKeyword>();
case Hash(u8"case"):
return new LexTokenImpl<LexTokenKind::CaseKeyword>();
case Hash(u8"cast"):
return new LexTokenImpl<LexTokenKind::CastKeyword>();
case Hash(u8"catch"):
return new LexTokenImpl<LexTokenKind::CatchKeyword>();
case Hash(u8"class"):
return new LexTokenImpl<LexTokenKind::ClassKeyword>();
case Hash(u8"const"):
return new LexTokenImpl<LexTokenKind::ConstKeyword>();
case Hash(u8"continue"):
return new LexTokenImpl<LexTokenKind::ContinueKeyword>();
case Hash(u8"default"):
return new LexTokenImpl<LexTokenKind::DefaultKeyword>();
case Hash(u8"do"):
return new LexTokenImpl<LexTokenKind::DoKeyword>();
case Hash(u8"double"):
return new LexTokenImpl<LexTokenKind::DoubleKeyword>();
case Hash(u8"else"):
return new LexTokenImpl<LexTokenKind::ElseKeyword>();
case Hash(u8"enum"):
return new LexTokenImpl<LexTokenKind::EnumKeyword>();
case Hash(u8"explicit"):
return new LexTokenImpl<LexTokenKind::ExplicitKeyword>();
case Hash(u8"external"):
return new LexTokenImpl<LexTokenKind::ExternalKeyword>();
case Hash(u8"false"):
return new LexTokenImpl<LexTokenKind::FalseKeyword>();
case Hash(u8"final"):
return new LexTokenImpl<LexTokenKind::FinalKeyword>();
case Hash(u8"float"):
return new LexTokenImpl<LexTokenKind::FloatKeyword>();
case Hash(u8"for"):
return new LexTokenImpl<LexTokenKind::ForKeyword>();
case Hash(u8"from"):
return new LexTokenImpl<LexTokenKind::FromKeyword>();
case Hash(u8"funcdef"):
return new LexTokenImpl<LexTokenKind::FuncdefKeyword>();
case Hash(u8"function"):
return new LexTokenImpl<LexTokenKind::FunctionKeyword>();
case Hash(u8"get"):
return new LexTokenImpl<LexTokenKind::GetKeyword>();
case Hash(u8"if"):
return new LexTokenImpl<LexTokenKind::IfKeyword>();
case Hash(u8"import"):
return new LexTokenImpl<LexTokenKind::ImportKeyword>();
case Hash(u8"in"):
return new LexTokenImpl<LexTokenKind::InKeyword>();
case Hash(u8"inout"):
return new LexTokenImpl<LexTokenKind::InoutKeyword>();
case Hash(u8"int"):
return new LexTokenImpl<LexTokenKind::IntKeyword>();
case Hash(u8"interface"):
return new LexTokenImpl<LexTokenKind::InterfaceKeyword>();
case Hash(u8"int8"):
return new LexTokenImpl<LexTokenKind::Int8Keyword>();
case Hash(u8"int16"):
return new LexTokenImpl<LexTokenKind::Int16Keyword>();
case Hash(u8"int32"):
return new LexTokenImpl<LexTokenKind::Int32Keyword>();
case Hash(u8"int64"):
return new LexTokenImpl<LexTokenKind::Int64Keyword>();
case Hash(u8"is"):
return new LexTokenImpl<LexTokenKind::IsKeyword>();
case Hash(u8"mixin"):
return new LexTokenImpl<LexTokenKind::MixinKeyword>();
case Hash(u8"namespace"):
return new LexTokenImpl<LexTokenKind::NamespaceKeyword>();
case Hash(u8"not"):
return new LexTokenImpl<LexTokenKind::NotKeyword>();
case Hash(u8"null"):
return new LexTokenImpl<LexTokenKind::NullKeyword>();
case Hash(u8"or"):
return new LexTokenImpl<LexTokenKind::OrKeyword>();
case Hash(u8"out"):
return new LexTokenImpl<LexTokenKind::OutKeyword>();
case Hash(u8"override"):
return new LexTokenImpl<LexTokenKind::OverrideKeyword>();
case Hash(u8"private"):
return new LexTokenImpl<LexTokenKind::PrivateKeyword>();
case Hash(u8"property"):
return new LexTokenImpl<LexTokenKind::PropertyKeyword>();
case Hash(u8"protected"):
return new LexTokenImpl<LexTokenKind::ProtectedKeyword>();
case Hash(u8"return"):
return new LexTokenImpl<LexTokenKind::ReturnKeyword>();
case Hash(u8"set"):
return new LexTokenImpl<LexTokenKind::SetKeyword>();
case Hash(u8"shared"):
return new LexTokenImpl<LexTokenKind::SharedKeyword>();
case Hash(u8"super"):
return new LexTokenImpl<LexTokenKind::SuperKeyword>();
case Hash(u8"switch"):
return new LexTokenImpl<LexTokenKind::SwitchKeyword>();
case Hash(u8"this"):
return new LexTokenImpl<LexTokenKind::ThisKeyword>();
case Hash(u8"true"):
return new LexTokenImpl<LexTokenKind::TrueKeyword>();
case Hash(u8"try"):
return new LexTokenImpl<LexTokenKind::TryKeyword>();
case Hash(u8"typedef"):
return new LexTokenImpl<LexTokenKind::TypedefKeyword>();
case Hash(u8"uint"):
return new LexTokenImpl<LexTokenKind::UintKeyword>();
case Hash(u8"uint8"):
return new LexTokenImpl<LexTokenKind::Uint8Keyword>();
case Hash(u8"uint16"):
return new LexTokenImpl<LexTokenKind::Uint16Keyword>();
case Hash(u8"uint32"):
return new LexTokenImpl<LexTokenKind::Uint32Keyword>();
case Hash(u8"uint64"):
return new LexTokenImpl<LexTokenKind::Uint64Keyword>();
case Hash(u8"void"):
return new LexTokenImpl<LexTokenKind::VoidKeyword>();
case Hash(u8"while"):
return new LexTokenImpl<LexTokenKind::WhileKeyword>();
case Hash(u8"xor"):
return new LexTokenImpl<LexTokenKind::XorKeyword>();
default:
return new IdentifierToken(std::u8string(str));
}
}
bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) {
if (c >= 'a' && c <= 'z') {
return true;
}
if (c >= 'A' && c <= 'Z') {
return true;
}
if (c >= '0' && c <= '9') {
return true;
}
if (c == '_') {
return true;
}
return false;
}
}
src/Parser/Lexer/Lexer.hpp
+3
View File
@@ -43,6 +43,9 @@ namespace ElohimScript::Parser {
IntegerLiteral* LexBinary();
StringLiteral* LexString(char8_t opening, bool heredoc);
LexToken* LexKeywordOrIdentifier();
static bool IsAlphaNumericalOrUnderscore(char8_t c);
};
}
tests/LexerTests/IdentifierLexTests.cpp
+108
View File
@@ -0,0 +1,108 @@
#include "../../extern/doctest.hpp"
#include "../../src/Parser/Lexer/Lexer.hpp"
using namespace ElohimScript::Parser;
#define KEYWORD_TEST(script, symbol) \
TEST_CASE("Lex " script) { \
auto lexer = Lexer(script); \
const auto* token = lexer.Lex(); \
CHECK(token->GetKind() == LexTokenKind::symbol); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
KEYWORD_TEST("and", AndKeyword);
KEYWORD_TEST("abstract", AbstractKeyword);
KEYWORD_TEST("auto", AutoKeyword);
KEYWORD_TEST("bool", BoolKeyword);
KEYWORD_TEST("break", BreakKeyword);
KEYWORD_TEST("case", CaseKeyword);
KEYWORD_TEST("cast", CastKeyword);
KEYWORD_TEST("catch", CatchKeyword);
KEYWORD_TEST("class", ClassKeyword);
KEYWORD_TEST("const", ConstKeyword);
KEYWORD_TEST("continue", ContinueKeyword);
KEYWORD_TEST("default", DefaultKeyword);
KEYWORD_TEST("do", DoKeyword);
KEYWORD_TEST("double", DoubleKeyword);
KEYWORD_TEST("else", ElseKeyword);
KEYWORD_TEST("enum", EnumKeyword);
KEYWORD_TEST("explicit", ExplicitKeyword);
KEYWORD_TEST("external", ExternalKeyword);
KEYWORD_TEST("false", FalseKeyword);
KEYWORD_TEST("final", FinalKeyword);
KEYWORD_TEST("float", FloatKeyword);
KEYWORD_TEST("for", ForKeyword);
KEYWORD_TEST("from", FromKeyword);
KEYWORD_TEST("funcdef", FuncdefKeyword);
KEYWORD_TEST("function", FunctionKeyword);
KEYWORD_TEST("get", GetKeyword);
KEYWORD_TEST("if", IfKeyword);
KEYWORD_TEST("import", ImportKeyword);
KEYWORD_TEST("in", InKeyword);
KEYWORD_TEST("inout", InoutKeyword);
KEYWORD_TEST("int", IntKeyword);
KEYWORD_TEST("interface", InterfaceKeyword);
KEYWORD_TEST("int8", Int8Keyword);
KEYWORD_TEST("int16", Int16Keyword);
KEYWORD_TEST("int32", Int32Keyword);
KEYWORD_TEST("int64", Int64Keyword);
KEYWORD_TEST("is", IsKeyword);
KEYWORD_TEST("mixin", MixinKeyword);
KEYWORD_TEST("namespace", NamespaceKeyword);
KEYWORD_TEST("not", NotKeyword);
KEYWORD_TEST("null", NullKeyword);
KEYWORD_TEST("or", OrKeyword);
KEYWORD_TEST("out", OutKeyword);
KEYWORD_TEST("override", OverrideKeyword);
KEYWORD_TEST("private", PrivateKeyword);
KEYWORD_TEST("property", PropertyKeyword);
KEYWORD_TEST("protected", ProtectedKeyword);
KEYWORD_TEST("return", ReturnKeyword);
KEYWORD_TEST("set", SetKeyword);
KEYWORD_TEST("shared", SharedKeyword);
KEYWORD_TEST("super", SuperKeyword);
KEYWORD_TEST("switch", SwitchKeyword);
KEYWORD_TEST("this", ThisKeyword);
KEYWORD_TEST("true", TrueKeyword);
KEYWORD_TEST("try", TryKeyword);
KEYWORD_TEST("typedef", TypedefKeyword);
KEYWORD_TEST("uint", UintKeyword);
KEYWORD_TEST("uint8", Uint8Keyword);
KEYWORD_TEST("uint16", Uint16Keyword);
KEYWORD_TEST("uint32", Uint32Keyword);
KEYWORD_TEST("uint64", Uint64Keyword);
KEYWORD_TEST("void", VoidKeyword);
KEYWORD_TEST("while", WhileKeyword);
KEYWORD_TEST("xor", XorKeyword);
namespace doctest {
template <> struct StringMaker<std::u8string> {
static String convert(const std::u8string& value) {
return String(reinterpret_cast<const char*>(value.data()));
}
};
template <> struct StringMaker<LexTokenKind> {
static String convert(LexTokenKind value) { return String(std::to_string((uint32_t)value).c_str()); }
};
}
#define IDENTIFIER_TEST(identifier) \
TEST_CASE("Lex identifier " identifier) { \
auto lexer = Lexer(identifier); \
const auto* token = lexer.Lex(); \
REQUIRE(token->GetKind() == LexTokenKind::Identifier); \
auto value = ((IdentifierToken*)token)->GetValue(); \
CHECK(value == std::u8string(reinterpret_cast<const char8_t*>(identifier))); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
IDENTIFIER_TEST("foobar");
IDENTIFIER_TEST("_foobar");
IDENTIFIER_TEST("_foo8bar");
#undef KEYWORD_TEST
#undef IDENTIFIER_TEST