Deukhoofd/MalachScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lex identifiers and keywords.

Browse Source
This commit is contained in:
Deukhoofd
2020-10-04 18:30:53 +02:00
parent db7ad0bd76
commit 20976010d6
5 changed files with 352 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/Parser/Lexer/LexToken.hpp
View File

@@ -46,6 +46,14 @@ namespace ElohimScript::Parser {
StringLiteral(std::u8string value) : _value(std::move(value)) {}
[[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; }
};
class IdentifierToken : public LexTokenImpl<LexTokenKind::Identifier>{
std::u8string _value;
public:
IdentifierToken(std::u8string value) : _value(std::move(value)) {}
[[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; }
};
}
#endif // ELOHIMSCRIPT_LEXTOKEN_HPP

67
src/Parser/Lexer/LexTokenKind.hpp
View File

@@ -62,10 +62,77 @@ namespace ElohimScript::Parser {
ExclamationMarkIsSymbol,
ColonColonSymbol,
// Keywords
AndKeyword,
AbstractKeyword,
AutoKeyword,
BoolKeyword,
BreakKeyword,
CaseKeyword,
CastKeyword,
CatchKeyword,
ClassKeyword,
ConstKeyword,
ContinueKeyword,
DefaultKeyword,
DoKeyword,
DoubleKeyword,
ElseKeyword,
EnumKeyword,
ExplicitKeyword,
ExternalKeyword,
FalseKeyword,
FinalKeyword,
FloatKeyword,
ForKeyword,
FromKeyword,
FuncdefKeyword,
FunctionKeyword,
GetKeyword,
IfKeyword,
ImportKeyword,
InKeyword,
InoutKeyword,
IntKeyword,
InterfaceKeyword,
Int8Keyword,
Int16Keyword,
Int32Keyword,
Int64Keyword,
IsKeyword,
MixinKeyword,
NamespaceKeyword,
NotKeyword,
NullKeyword,
OrKeyword,
OutKeyword,
OverrideKeyword,
PrivateKeyword,
PropertyKeyword,
ProtectedKeyword,
ReturnKeyword,
SetKeyword,
SharedKeyword,
SuperKeyword,
SwitchKeyword,
ThisKeyword,
TrueKeyword,
TryKeyword,
TypedefKeyword,
UintKeyword,
Uint8Keyword,
Uint16Keyword,
Uint32Keyword,
Uint64Keyword,
VoidKeyword,
WhileKeyword,
XorKeyword,
// Literals
FloatLiteral,
IntegerLiteral,
StringLiteral,
Identifier,
};
}

167
src/Parser/Lexer/Lexer.cpp
View File

@@ -223,7 +223,11 @@ namespace ElohimScript::Parser {
return LexString(u8'"', false);
}
default: return new LexTokenImpl<LexTokenKind::Unknown>();
default:
if (IsAlphaNumericalOrUnderscore(c))
return LexKeywordOrIdentifier();
// TODO: Log error
return new LexTokenImpl<LexTokenKind::Unknown>();
}
}
@@ -399,4 +403,165 @@ namespace ElohimScript::Parser {
}
return new StringLiteral(std::u8string(_script.substr(start, offset)));
}
static uint32_t constexpr Hash(const char8_t* input) {
return *input ? static_cast<uint32_t>(*input) + 33 * Hash(input + 1) : 5381;
};
LexToken* Lexer::LexKeywordOrIdentifier() {
auto offset = 0;
while (IsAlphaNumericalOrUnderscore(Peek(offset))) {
offset++;
}
auto str = _script.substr(_position, offset);
Progress(offset);
switch (Hash(str.data())) {
case Hash(u8"and"):
return new LexTokenImpl<LexTokenKind::AndKeyword>();
case Hash(u8"abstract"):
return new LexTokenImpl<LexTokenKind::AbstractKeyword>();
case Hash(u8"auto"):
return new LexTokenImpl<LexTokenKind::AutoKeyword>();
case Hash(u8"bool"):
return new LexTokenImpl<LexTokenKind::BoolKeyword>();
case Hash(u8"break"):
return new LexTokenImpl<LexTokenKind::BreakKeyword>();
case Hash(u8"case"):
return new LexTokenImpl<LexTokenKind::CaseKeyword>();
case Hash(u8"cast"):
return new LexTokenImpl<LexTokenKind::CastKeyword>();
case Hash(u8"catch"):
return new LexTokenImpl<LexTokenKind::CatchKeyword>();
case Hash(u8"class"):
return new LexTokenImpl<LexTokenKind::ClassKeyword>();
case Hash(u8"const"):
return new LexTokenImpl<LexTokenKind::ConstKeyword>();
case Hash(u8"continue"):
return new LexTokenImpl<LexTokenKind::ContinueKeyword>();
case Hash(u8"default"):
return new LexTokenImpl<LexTokenKind::DefaultKeyword>();
case Hash(u8"do"):
return new LexTokenImpl<LexTokenKind::DoKeyword>();
case Hash(u8"double"):
return new LexTokenImpl<LexTokenKind::DoubleKeyword>();
case Hash(u8"else"):
return new LexTokenImpl<LexTokenKind::ElseKeyword>();
case Hash(u8"enum"):
return new LexTokenImpl<LexTokenKind::EnumKeyword>();
case Hash(u8"explicit"):
return new LexTokenImpl<LexTokenKind::ExplicitKeyword>();
case Hash(u8"external"):
return new LexTokenImpl<LexTokenKind::ExternalKeyword>();
case Hash(u8"false"):
return new LexTokenImpl<LexTokenKind::FalseKeyword>();
case Hash(u8"final"):
return new LexTokenImpl<LexTokenKind::FinalKeyword>();
case Hash(u8"float"):
return new LexTokenImpl<LexTokenKind::FloatKeyword>();
case Hash(u8"for"):
return new LexTokenImpl<LexTokenKind::ForKeyword>();
case Hash(u8"from"):
return new LexTokenImpl<LexTokenKind::FromKeyword>();
case Hash(u8"funcdef"):
return new LexTokenImpl<LexTokenKind::FuncdefKeyword>();
case Hash(u8"function"):
return new LexTokenImpl<LexTokenKind::FunctionKeyword>();
case Hash(u8"get"):
return new LexTokenImpl<LexTokenKind::GetKeyword>();
case Hash(u8"if"):
return new LexTokenImpl<LexTokenKind::IfKeyword>();
case Hash(u8"import"):
return new LexTokenImpl<LexTokenKind::ImportKeyword>();
case Hash(u8"in"):
return new LexTokenImpl<LexTokenKind::InKeyword>();
case Hash(u8"inout"):
return new LexTokenImpl<LexTokenKind::InoutKeyword>();
case Hash(u8"int"):
return new LexTokenImpl<LexTokenKind::IntKeyword>();
case Hash(u8"interface"):
return new LexTokenImpl<LexTokenKind::InterfaceKeyword>();
case Hash(u8"int8"):
return new LexTokenImpl<LexTokenKind::Int8Keyword>();
case Hash(u8"int16"):
return new LexTokenImpl<LexTokenKind::Int16Keyword>();
case Hash(u8"int32"):
return new LexTokenImpl<LexTokenKind::Int32Keyword>();
case Hash(u8"int64"):
return new LexTokenImpl<LexTokenKind::Int64Keyword>();
case Hash(u8"is"):
return new LexTokenImpl<LexTokenKind::IsKeyword>();
case Hash(u8"mixin"):
return new LexTokenImpl<LexTokenKind::MixinKeyword>();
case Hash(u8"namespace"):
return new LexTokenImpl<LexTokenKind::NamespaceKeyword>();
case Hash(u8"not"):
return new LexTokenImpl<LexTokenKind::NotKeyword>();
case Hash(u8"null"):
return new LexTokenImpl<LexTokenKind::NullKeyword>();
case Hash(u8"or"):
return new LexTokenImpl<LexTokenKind::OrKeyword>();
case Hash(u8"out"):
return new LexTokenImpl<LexTokenKind::OutKeyword>();
case Hash(u8"override"):
return new LexTokenImpl<LexTokenKind::OverrideKeyword>();
case Hash(u8"private"):
return new LexTokenImpl<LexTokenKind::PrivateKeyword>();
case Hash(u8"property"):
return new LexTokenImpl<LexTokenKind::PropertyKeyword>();
case Hash(u8"protected"):
return new LexTokenImpl<LexTokenKind::ProtectedKeyword>();
case Hash(u8"return"):
return new LexTokenImpl<LexTokenKind::ReturnKeyword>();
case Hash(u8"set"):
return new LexTokenImpl<LexTokenKind::SetKeyword>();
case Hash(u8"shared"):
return new LexTokenImpl<LexTokenKind::SharedKeyword>();
case Hash(u8"super"):
return new LexTokenImpl<LexTokenKind::SuperKeyword>();
case Hash(u8"switch"):
return new LexTokenImpl<LexTokenKind::SwitchKeyword>();
case Hash(u8"this"):
return new LexTokenImpl<LexTokenKind::ThisKeyword>();
case Hash(u8"true"):
return new LexTokenImpl<LexTokenKind::TrueKeyword>();
case Hash(u8"try"):
return new LexTokenImpl<LexTokenKind::TryKeyword>();
case Hash(u8"typedef"):
return new LexTokenImpl<LexTokenKind::TypedefKeyword>();
case Hash(u8"uint"):
return new LexTokenImpl<LexTokenKind::UintKeyword>();
case Hash(u8"uint8"):
return new LexTokenImpl<LexTokenKind::Uint8Keyword>();
case Hash(u8"uint16"):
return new LexTokenImpl<LexTokenKind::Uint16Keyword>();
case Hash(u8"uint32"):
return new LexTokenImpl<LexTokenKind::Uint32Keyword>();
case Hash(u8"uint64"):
return new LexTokenImpl<LexTokenKind::Uint64Keyword>();
case Hash(u8"void"):
return new LexTokenImpl<LexTokenKind::VoidKeyword>();
case Hash(u8"while"):
return new LexTokenImpl<LexTokenKind::WhileKeyword>();
case Hash(u8"xor"):
return new LexTokenImpl<LexTokenKind::XorKeyword>();
default:
return new IdentifierToken(std::u8string(str));
}
}
bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) {
if (c >= 'a' && c <= 'z') {
return true;
}
if (c >= 'A' && c <= 'Z') {
return true;
}
if (c >= '0' && c <= '9') {
return true;
}
if (c == '_') {
return true;
}
return false;
}
}

3
src/Parser/Lexer/Lexer.hpp
View File

@@ -43,6 +43,9 @@ namespace ElohimScript::Parser {
IntegerLiteral* LexBinary();
StringLiteral* LexString(char8_t opening, bool heredoc);
LexToken* LexKeywordOrIdentifier();
static bool IsAlphaNumericalOrUnderscore(char8_t c);
};
}