Deukhoofd/MalachScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use block allocator for lexer.

Browse Source
This commit is contained in:
Deukhoofd 2020-10-04 22:21:20 +02:00
parent 0ce2feee06
commit 739e2e6f17
Signed by: Deukhoofd
GPG Key ID: F63E044490819F6F
9 changed files with 206 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/Parser/Lexer/LexResult.hpp Normal file
View File

@ -0,0 +1,19 @@
#ifndef ELOHIMSCRIPT_LEXRESULT_HPP
#define ELOHIMSCRIPT_LEXRESULT_HPP
#include "LexToken.hpp"
namespace ElohimScript::Parser {
class LexResult {
public:
LexResult(const LexToken* first, const uint8_t* memoryMap) : _first(first), _memoryMap(memoryMap) {}
[[nodiscard]] const LexToken* GetFirst() const noexcept { return _first; }
[[nodiscard]] const uint8_t* GetMemoryMap() const noexcept { return _memoryMap; }
private:
const LexToken* _first;
const uint8_t* _memoryMap;
};
}
#endif // ELOHIMSCRIPT_LEXRESULT_HPP

254
src/Parser/Lexer/Lexer.cpp
View File

@ -25,7 +25,7 @@ namespace ElohimScript::Parser {
auto start = _position;
auto c = Consume();
switch (c) {
case u8'\0': return new LexTokenImpl<LexTokenKind::EndOfFile>(TextSpan(start, 1));
case u8'\0': return Create<LexTokenImpl<LexTokenKind::EndOfFile>>(TextSpan(start, 1));
case u8'*': {
auto n = Peek();
if (n == u8'*') {
@ -34,91 +34,91 @@ namespace ElohimScript::Parser {
if (n == u8'=') {
Progress();
// **=
return new LexTokenImpl<LexTokenKind::StarStarEqualsSymbol>(TextSpan(start, 3));
return Create<LexTokenImpl<LexTokenKind::StarStarEqualsSymbol>>(TextSpan(start, 3));
}
// **
return new LexTokenImpl<LexTokenKind::StarStarSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::StarStarSymbol>>(TextSpan(start, 2));
}
if (n == u8'=') {
Progress();
// *=
return new LexTokenImpl<LexTokenKind::StarEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::StarEqualsSymbol>>(TextSpan(start, 2));
}
// *
return new LexTokenImpl<LexTokenKind::StarSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::StarSymbol>>(TextSpan(start, 1));
}
case u8'/':
if (Peek() == u8'=') {
Progress();
// /=
return new LexTokenImpl<LexTokenKind::SlashEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::SlashEqualsSymbol>>(TextSpan(start, 2));
}
// /
return new LexTokenImpl<LexTokenKind::SlashSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::SlashSymbol>>(TextSpan(start, 1));
case u8'%':
if (Peek() == u8'=') {
Progress();
// %=
return new LexTokenImpl<LexTokenKind::PercentEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::PercentEqualsSymbol>>(TextSpan(start, 2));
}
// %
return new LexTokenImpl<LexTokenKind::PercentSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::PercentSymbol>>(TextSpan(start, 1));
case u8'+': {
auto n = Peek();
if (n == u8'=') {
Progress();
// +=
return new LexTokenImpl<LexTokenKind::PlusEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::PlusEqualsSymbol>>(TextSpan(start, 2));
}
if (n == u8'+') {
Progress();
// ++
return new LexTokenImpl<LexTokenKind::PlusPlusSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::PlusPlusSymbol>>(TextSpan(start, 2));
}
// +
return new LexTokenImpl<LexTokenKind::PlusSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::PlusSymbol>>(TextSpan(start, 1));
}
case u8'-': {
auto n = Peek();
if (n == u8'=') {
Progress();
// -=
return new LexTokenImpl<LexTokenKind::MinusEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::MinusEqualsSymbol>>(TextSpan(start, 2));
}
if (n == u8'-') {
Progress();
// --
return new LexTokenImpl<LexTokenKind::MinusMinusSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::MinusMinusSymbol>>(TextSpan(start, 2));
}
// -
return new LexTokenImpl<LexTokenKind::MinusSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::MinusSymbol>>(TextSpan(start, 1));
}
case u8'<': {
auto n = Peek();
if (n == u8'=') {
Progress();
// <=
return new LexTokenImpl<LexTokenKind::LessThanEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::LessThanEqualsSymbol>>(TextSpan(start, 2));
}
if (n == u8'<') {
Progress();
if (Peek() == u8'=') {
Progress();
// <<=
return new LexTokenImpl<LexTokenKind::LessThanLessThanEqualsSymbol>(TextSpan(start, 3));
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanEqualsSymbol>>(TextSpan(start, 3));
}
// <<
return new LexTokenImpl<LexTokenKind::LessThanLessThanSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanSymbol>>(TextSpan(start, 2));
}
// <
return new LexTokenImpl<LexTokenKind::LessThanSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::LessThanSymbol>>(TextSpan(start, 1));
}
case u8'>': {
auto n = Peek();
if (n == u8'=') {
Progress();
// >=
return new LexTokenImpl<LexTokenKind::GreaterThanEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::GreaterThanEqualsSymbol>>(TextSpan(start, 2));
}
if (n == u8'>') {
Progress();
@ -126,126 +126,126 @@ namespace ElohimScript::Parser {
if (n == u8'=') {
Progress();
// >>=
return new LexTokenImpl<LexTokenKind::GreaterThanGreaterThanEqualsSymbol>(TextSpan(start, 3));
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanEqualsSymbol>>(TextSpan(start, 3));
}
if (n == u8'>') {
Progress();
if (Peek() == u8'=') {
Progress();
// >>>=
return new LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanEqualsSymbol>(
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanEqualsSymbol>>(
TextSpan(start, 4));
}
// >>>
return new LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanSymbol>(
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanSymbol>>(
TextSpan(start, 3));
}
// >>
return new LexTokenImpl<LexTokenKind::GreaterThanGreaterThanSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanSymbol>>(TextSpan(start, 2));
}
// >
return new LexTokenImpl<LexTokenKind::GreaterThanSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::GreaterThanSymbol>>(TextSpan(start, 1));
}
case u8'(': return new LexTokenImpl<LexTokenKind::OpenParenthesisSymbol>(TextSpan(start, 1));
case u8')': return new LexTokenImpl<LexTokenKind::CloseParenthesisSymbol>(TextSpan(start, 1));
case u8'(': return Create<LexTokenImpl<LexTokenKind::OpenParenthesisSymbol>>(TextSpan(start, 1));
case u8')': return Create<LexTokenImpl<LexTokenKind::CloseParenthesisSymbol>>(TextSpan(start, 1));
case u8'=': {
if (Peek() == u8'=') {
Progress();
// ==
return new LexTokenImpl<LexTokenKind::EqualsEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::EqualsEqualsSymbol>>(TextSpan(start, 2));
}
// =
return new LexTokenImpl<LexTokenKind::EqualsSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::EqualsSymbol>>(TextSpan(start, 1));
}
case u8'!': {
auto n = Peek();
if (n == u8'=') {
Progress();
// !=
return new LexTokenImpl<LexTokenKind::ExclamationMarkEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkEqualsSymbol>>(TextSpan(start, 2));
}
if (n == u8'i' && Peek(2) == u8's') {
Progress(2);
// !is
return new LexTokenImpl<LexTokenKind::ExclamationMarkIsSymbol>(TextSpan(start, 3));
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkIsSymbol>>(TextSpan(start, 3));
}
// !
return new LexTokenImpl<LexTokenKind::ExclamationMarkSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkSymbol>>(TextSpan(start, 1));
}
case u8'?': return new LexTokenImpl<LexTokenKind::QuestionMarkSymbol>(TextSpan(start, 1));
case u8'?': return Create<LexTokenImpl<LexTokenKind::QuestionMarkSymbol>>(TextSpan(start, 1));
case u8':': {
if (Peek() == u8':') {
Progress();
// ::
return new LexTokenImpl<LexTokenKind::ColonColonSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::ColonColonSymbol>>(TextSpan(start, 2));
}
// :
return new LexTokenImpl<LexTokenKind::ColonSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::ColonSymbol>>(TextSpan(start, 1));
}
case u8'&': {
auto n = Peek();
if (n == u8'=') {
Progress();
// &=
return new LexTokenImpl<LexTokenKind::AmpersandEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::AmpersandEqualsSymbol>>(TextSpan(start, 2));
}
if (n == u8'&') {
Progress();
// &&
return new LexTokenImpl<LexTokenKind::AmpersandAmpersandSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::AmpersandAmpersandSymbol>>(TextSpan(start, 2));
}
// &
return new LexTokenImpl<LexTokenKind::AmpersandSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::AmpersandSymbol>>(TextSpan(start, 1));
}
case u8',': return new LexTokenImpl<LexTokenKind::CommaSymbol>(TextSpan(start, 1));
case u8'{': return new LexTokenImpl<LexTokenKind::OpenCurlyParenthesisSymbol>(TextSpan(start, 1));
case u8'}': return new LexTokenImpl<LexTokenKind::CloseCurlyParenthesisSymbol>(TextSpan(start, 1));
case u8';': return new LexTokenImpl<LexTokenKind::SemicolonSymbol>(TextSpan(start, 1));
case u8',': return Create<LexTokenImpl<LexTokenKind::CommaSymbol>>(TextSpan(start, 1));
case u8'{': return Create<LexTokenImpl<LexTokenKind::OpenCurlyParenthesisSymbol>>(TextSpan(start, 1));
case u8'}': return Create<LexTokenImpl<LexTokenKind::CloseCurlyParenthesisSymbol>>(TextSpan(start, 1));
case u8';': return Create<LexTokenImpl<LexTokenKind::SemicolonSymbol>>(TextSpan(start, 1));
case u8'|': {
auto n = Peek();
if (n == u8'=') {
Progress();
// |=
return new LexTokenImpl<LexTokenKind::VerticalLineEqualsSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::VerticalLineEqualsSymbol>>(TextSpan(start, 2));
}
if (n == u8'|') {
Progress();
// ||
return new LexTokenImpl<LexTokenKind::VerticalLineVerticalLineSymbol>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::VerticalLineVerticalLineSymbol>>(TextSpan(start, 2));
}
// |
return new LexTokenImpl<LexTokenKind::VerticalLineSymbol>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::VerticalLineSymbol>>(TextSpan(start, 1));
}
case u8'^': {
auto n = Peek();
if (n == u8'=') {
Progress();
// ^=
return new LexTokenImpl<LexTokenKind::CaretEqualsSymbol>(TextSpan(start, start + 2));
return Create<LexTokenImpl<LexTokenKind::CaretEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'^') {
Progress();
// ^^
return new LexTokenImpl<LexTokenKind::CaretCaretSymbol>(TextSpan(start, start + 2));
return Create<LexTokenImpl<LexTokenKind::CaretCaretSymbol>>(TextSpan(start, start + 2));
}
// ^
return new LexTokenImpl<LexTokenKind::CaretSymbol>(TextSpan(start, start + 1));
return Create<LexTokenImpl<LexTokenKind::CaretSymbol>>(TextSpan(start, start + 1));
}
case u8'~': return new LexTokenImpl<LexTokenKind::TildeSymbol>(TextSpan(start, start + 1));
case u8'.': return new LexTokenImpl<LexTokenKind::DotSymbol>(TextSpan(start, start + 1));
case u8'[': return new LexTokenImpl<LexTokenKind::OpenBlockParenthesisSymbol>(TextSpan(start, start + 1));
case u8']': return new LexTokenImpl<LexTokenKind::CloseBlockParenthesisSymbol>(TextSpan(start, start + 1));
case u8'@': return new LexTokenImpl<LexTokenKind::AtSymbol>(TextSpan(start, start + 1));
case u8'~': return Create<LexTokenImpl<LexTokenKind::TildeSymbol>>(TextSpan(start, start + 1));
case u8'.': return Create<LexTokenImpl<LexTokenKind::DotSymbol>>(TextSpan(start, start + 1));
case u8'[': return Create<LexTokenImpl<LexTokenKind::OpenBlockParenthesisSymbol>>(TextSpan(start, start + 1));
case u8']': return Create<LexTokenImpl<LexTokenKind::CloseBlockParenthesisSymbol>>(TextSpan(start, start + 1));
case u8'@': return Create<LexTokenImpl<LexTokenKind::AtSymbol>>(TextSpan(start, start + 1));
case u8' ':
case u8'\r':
case u8'\n':
case u8'\t': return new LexTokenImpl<LexTokenKind::Whitespace>(TextSpan(start, start + 1));
case u8'\t': return Create<LexTokenImpl<LexTokenKind::Whitespace>>(TextSpan(start, start + 1));
// Byte order mark
case u8'\xEF': {
if (Peek() == u8'\xBB' && Peek(2) == u8'\xBF') {
Progress(2);
return new LexTokenImpl<LexTokenKind::Whitespace>(TextSpan(start, start + 3));
return Create<LexTokenImpl<LexTokenKind::Whitespace>>(TextSpan(start, start + 3));
}
}
case u8'0':
@ -270,7 +270,7 @@ namespace ElohimScript::Parser {
if (IsAlphaNumericalOrUnderscore(c))
return LexKeywordOrIdentifier();
_diagnostics->LogError(Diagnostics::DiagnosticType::UnknownToken, TextSpan(start, start + 1));
return new LexTokenImpl<LexTokenKind::Unknown>(TextSpan(start, start + 1));
return Create<LexTokenImpl<LexTokenKind::Unknown>>(TextSpan(start, start + 1));
}
}
@ -374,9 +374,9 @@ namespace ElohimScript::Parser {
if (isExponent) {
val *= pow(10, exponentValue);
}
return new FloatLiteral(TextSpan(start, _position), val);
return Create<FloatLiteral>(TextSpan(start, _position), val);
}
return new IntegerLiteral(TextSpan(start, _position), value);
return Create<IntegerLiteral>(TextSpan(start, _position), value);
}
IntegerLiteral* Lexer::LexHexadecimal() {
@ -391,7 +391,7 @@ namespace ElohimScript::Parser {
value <<= 4;
value += v;
}
return new IntegerLiteral(TextSpan(start, _position), value);
return Create<IntegerLiteral>(TextSpan(start, _position), value);
}
IntegerLiteral* Lexer::LexOctal() {
auto start = _position;
@ -405,7 +405,7 @@ namespace ElohimScript::Parser {
value <<= 3;
value += v;
}
return new IntegerLiteral(TextSpan(start, _position), value);
return Create<IntegerLiteral>(TextSpan(start, _position), value);
}
IntegerLiteral* Lexer::LexBinary() {
auto start = _position;
@ -419,7 +419,7 @@ namespace ElohimScript::Parser {
value <<= 1;
value += v;
}
return new IntegerLiteral(TextSpan(start, _position), value);
return Create<IntegerLiteral>(TextSpan(start, _position), value);
}
StringLiteral* Lexer::LexString(char8_t opening, bool heredoc) {
Progress();
@ -453,7 +453,7 @@ namespace ElohimScript::Parser {
if (heredoc) {
Progress(2);
}
return new StringLiteral(TextSpan(start, start + _position), std::u8string(_script.substr(start, offset)));
return Create<StringLiteral>(TextSpan(start, start + _position), std::u8string(_script.substr(start, offset)));
}
static uint32_t constexpr Hash(const char8_t* input) {
@ -469,75 +469,75 @@ namespace ElohimScript::Parser {
auto str = std::u8string(_script.substr(start, offset));
Progress(offset - 1);
switch (Hash(str.c_str())) {
case Hash(u8"and"): return new LexTokenImpl<LexTokenKind::AndKeyword>(TextSpan(start, _position));
case Hash(u8"abstract"): return new LexTokenImpl<LexTokenKind::AbstractKeyword>(TextSpan(start, _position));
case Hash(u8"auto"): return new LexTokenImpl<LexTokenKind::AutoKeyword>(TextSpan(start, _position));
case Hash(u8"bool"): return new LexTokenImpl<LexTokenKind::BoolKeyword>(TextSpan(start, _position));
case Hash(u8"break"): return new LexTokenImpl<LexTokenKind::BreakKeyword>(TextSpan(start, _position));
case Hash(u8"case"): return new LexTokenImpl<LexTokenKind::CaseKeyword>(TextSpan(start, _position));
case Hash(u8"cast"): return new LexTokenImpl<LexTokenKind::CastKeyword>(TextSpan(start, _position));
case Hash(u8"catch"): return new LexTokenImpl<LexTokenKind::CatchKeyword>(TextSpan(start, _position));
case Hash(u8"class"): return new LexTokenImpl<LexTokenKind::ClassKeyword>(TextSpan(start, _position));
case Hash(u8"const"): return new LexTokenImpl<LexTokenKind::ConstKeyword>(TextSpan(start, _position));
case Hash(u8"continue"): return new LexTokenImpl<LexTokenKind::ContinueKeyword>(TextSpan(start, _position));
case Hash(u8"default"): return new LexTokenImpl<LexTokenKind::DefaultKeyword>(TextSpan(start, _position));
case Hash(u8"do"): return new LexTokenImpl<LexTokenKind::DoKeyword>(TextSpan(start, _position));
case Hash(u8"double"): return new LexTokenImpl<LexTokenKind::DoubleKeyword>(TextSpan(start, _position));
case Hash(u8"else"): return new LexTokenImpl<LexTokenKind::ElseKeyword>(TextSpan(start, _position));
case Hash(u8"enum"): return new LexTokenImpl<LexTokenKind::EnumKeyword>(TextSpan(start, _position));
case Hash(u8"explicit"): return new LexTokenImpl<LexTokenKind::ExplicitKeyword>(TextSpan(start, _position));
case Hash(u8"external"): return new LexTokenImpl<LexTokenKind::ExternalKeyword>(TextSpan(start, _position));
case Hash(u8"false"): return new LexTokenImpl<LexTokenKind::FalseKeyword>(TextSpan(start, _position));
case Hash(u8"final"): return new LexTokenImpl<LexTokenKind::FinalKeyword>(TextSpan(start, _position));
case Hash(u8"float"): return new LexTokenImpl<LexTokenKind::FloatKeyword>(TextSpan(start, _position));
case Hash(u8"for"): return new LexTokenImpl<LexTokenKind::ForKeyword>(TextSpan(start, _position));
case Hash(u8"from"): return new LexTokenImpl<LexTokenKind::FromKeyword>(TextSpan(start, _position));
case Hash(u8"funcdef"): return new LexTokenImpl<LexTokenKind::FuncdefKeyword>(TextSpan(start, _position));
case Hash(u8"function"): return new LexTokenImpl<LexTokenKind::FunctionKeyword>(TextSpan(start, _position));
case Hash(u8"get"): return new LexTokenImpl<LexTokenKind::GetKeyword>(TextSpan(start, _position));
case Hash(u8"if"): return new LexTokenImpl<LexTokenKind::IfKeyword>(TextSpan(start, _position));
case Hash(u8"import"): return new LexTokenImpl<LexTokenKind::ImportKeyword>(TextSpan(start, _position));
case Hash(u8"in"): return new LexTokenImpl<LexTokenKind::InKeyword>(TextSpan(start, _position));
case Hash(u8"inout"): return new LexTokenImpl<LexTokenKind::InoutKeyword>(TextSpan(start, _position));
case Hash(u8"int"): return new LexTokenImpl<LexTokenKind::IntKeyword>(TextSpan(start, _position));
case Hash(u8"and"): return Create<LexTokenImpl<LexTokenKind::AndKeyword>>(TextSpan(start, _position));
case Hash(u8"abstract"): return Create<LexTokenImpl<LexTokenKind::AbstractKeyword>>(TextSpan(start, _position));
case Hash(u8"auto"): return Create<LexTokenImpl<LexTokenKind::AutoKeyword>>(TextSpan(start, _position));
case Hash(u8"bool"): return Create<LexTokenImpl<LexTokenKind::BoolKeyword>>(TextSpan(start, _position));
case Hash(u8"break"): return Create<LexTokenImpl<LexTokenKind::BreakKeyword>>(TextSpan(start, _position));
case Hash(u8"case"): return Create<LexTokenImpl<LexTokenKind::CaseKeyword>>(TextSpan(start, _position));
case Hash(u8"cast"): return Create<LexTokenImpl<LexTokenKind::CastKeyword>>(TextSpan(start, _position));
case Hash(u8"catch"): return Create<LexTokenImpl<LexTokenKind::CatchKeyword>>(TextSpan(start, _position));
case Hash(u8"class"): return Create<LexTokenImpl<LexTokenKind::ClassKeyword>>(TextSpan(start, _position));
case Hash(u8"const"): return Create<LexTokenImpl<LexTokenKind::ConstKeyword>>(TextSpan(start, _position));
case Hash(u8"continue"): return Create<LexTokenImpl<LexTokenKind::ContinueKeyword>>(TextSpan(start, _position));
case Hash(u8"default"): return Create<LexTokenImpl<LexTokenKind::DefaultKeyword>>(TextSpan(start, _position));
case Hash(u8"do"): return Create<LexTokenImpl<LexTokenKind::DoKeyword>>(TextSpan(start, _position));
case Hash(u8"double"): return Create<LexTokenImpl<LexTokenKind::DoubleKeyword>>(TextSpan(start, _position));
case Hash(u8"else"): return Create<LexTokenImpl<LexTokenKind::ElseKeyword>>(TextSpan(start, _position));
case Hash(u8"enum"): return Create<LexTokenImpl<LexTokenKind::EnumKeyword>>(TextSpan(start, _position));
case Hash(u8"explicit"): return Create<LexTokenImpl<LexTokenKind::ExplicitKeyword>>(TextSpan(start, _position));
case Hash(u8"external"): return Create<LexTokenImpl<LexTokenKind::ExternalKeyword>>(TextSpan(start, _position));
case Hash(u8"false"): return Create<LexTokenImpl<LexTokenKind::FalseKeyword>>(TextSpan(start, _position));
case Hash(u8"final"): return Create<LexTokenImpl<LexTokenKind::FinalKeyword>>(TextSpan(start, _position));
case Hash(u8"float"): return Create<LexTokenImpl<LexTokenKind::FloatKeyword>>(TextSpan(start, _position));
case Hash(u8"for"): return Create<LexTokenImpl<LexTokenKind::ForKeyword>>(TextSpan(start, _position));
case Hash(u8"from"): return Create<LexTokenImpl<LexTokenKind::FromKeyword>>(TextSpan(start, _position));
case Hash(u8"funcdef"): return Create<LexTokenImpl<LexTokenKind::FuncdefKeyword>>(TextSpan(start, _position));
case Hash(u8"function"): return Create<LexTokenImpl<LexTokenKind::FunctionKeyword>>(TextSpan(start, _position));
case Hash(u8"get"): return Create<LexTokenImpl<LexTokenKind::GetKeyword>>(TextSpan(start, _position));
case Hash(u8"if"): return Create<LexTokenImpl<LexTokenKind::IfKeyword>>(TextSpan(start, _position));
case Hash(u8"import"): return Create<LexTokenImpl<LexTokenKind::ImportKeyword>>(TextSpan(start, _position));
case Hash(u8"in"): return Create<LexTokenImpl<LexTokenKind::InKeyword>>(TextSpan(start, _position));
case Hash(u8"inout"): return Create<LexTokenImpl<LexTokenKind::InoutKeyword>>(TextSpan(start, _position));
case Hash(u8"int"): return Create<LexTokenImpl<LexTokenKind::IntKeyword>>(TextSpan(start, _position));
case Hash(u8"interface"):
return new LexTokenImpl<LexTokenKind::InterfaceKeyword>(TextSpan(start, _position));
case Hash(u8"int8"): return new LexTokenImpl<LexTokenKind::Int8Keyword>(TextSpan(start, _position));
case Hash(u8"int16"): return new LexTokenImpl<LexTokenKind::Int16Keyword>(TextSpan(start, _position));
case Hash(u8"int32"): return new LexTokenImpl<LexTokenKind::Int32Keyword>(TextSpan(start, _position));
case Hash(u8"int64"): return new LexTokenImpl<LexTokenKind::Int64Keyword>(TextSpan(start, _position));
case Hash(u8"is"): return new LexTokenImpl<LexTokenKind::IsKeyword>(TextSpan(start, _position));
case Hash(u8"mixin"): return new LexTokenImpl<LexTokenKind::MixinKeyword>(TextSpan(start, _position));
return Create<LexTokenImpl<LexTokenKind::InterfaceKeyword>>(TextSpan(start, _position));
case Hash(u8"int8"): return Create<LexTokenImpl<LexTokenKind::Int8Keyword>>(TextSpan(start, _position));
case Hash(u8"int16"): return Create<LexTokenImpl<LexTokenKind::Int16Keyword>>(TextSpan(start, _position));
case Hash(u8"int32"): return Create<LexTokenImpl<LexTokenKind::Int32Keyword>>(TextSpan(start, _position));
case Hash(u8"int64"): return Create<LexTokenImpl<LexTokenKind::Int64Keyword>>(TextSpan(start, _position));
case Hash(u8"is"): return Create<LexTokenImpl<LexTokenKind::IsKeyword>>(TextSpan(start, _position));
case Hash(u8"mixin"): return Create<LexTokenImpl<LexTokenKind::MixinKeyword>>(TextSpan(start, _position));
case Hash(u8"namespace"):
return new LexTokenImpl<LexTokenKind::NamespaceKeyword>(TextSpan(start, _position));
case Hash(u8"not"): return new LexTokenImpl<LexTokenKind::NotKeyword>(TextSpan(start, _position));
case Hash(u8"null"): return new LexTokenImpl<LexTokenKind::NullKeyword>(TextSpan(start, _position));
case Hash(u8"or"): return new LexTokenImpl<LexTokenKind::OrKeyword>(TextSpan(start, _position));
case Hash(u8"out"): return new LexTokenImpl<LexTokenKind::OutKeyword>(TextSpan(start, _position));
case Hash(u8"override"): return new LexTokenImpl<LexTokenKind::OverrideKeyword>(TextSpan(start, _position));
case Hash(u8"private"): return new LexTokenImpl<LexTokenKind::PrivateKeyword>(TextSpan(start, _position));
case Hash(u8"property"): return new LexTokenImpl<LexTokenKind::PropertyKeyword>(TextSpan(start, _position));
return Create<LexTokenImpl<LexTokenKind::NamespaceKeyword>>(TextSpan(start, _position));
case Hash(u8"not"): return Create<LexTokenImpl<LexTokenKind::NotKeyword>>(TextSpan(start, _position));
case Hash(u8"null"): return Create<LexTokenImpl<LexTokenKind::NullKeyword>>(TextSpan(start, _position));
case Hash(u8"or"): return Create<LexTokenImpl<LexTokenKind::OrKeyword>>(TextSpan(start, _position));
case Hash(u8"out"): return Create<LexTokenImpl<LexTokenKind::OutKeyword>>(TextSpan(start, _position));
case Hash(u8"override"): return Create<LexTokenImpl<LexTokenKind::OverrideKeyword>>(TextSpan(start, _position));
case Hash(u8"private"): return Create<LexTokenImpl<LexTokenKind::PrivateKeyword>>(TextSpan(start, _position));
case Hash(u8"property"): return Create<LexTokenImpl<LexTokenKind::PropertyKeyword>>(TextSpan(start, _position));
case Hash(u8"protected"):
return new LexTokenImpl<LexTokenKind::ProtectedKeyword>(TextSpan(start, _position));
case Hash(u8"return"): return new LexTokenImpl<LexTokenKind::ReturnKeyword>(TextSpan(start, _position));
case Hash(u8"set"): return new LexTokenImpl<LexTokenKind::SetKeyword>(TextSpan(start, _position));
case Hash(u8"shared"): return new LexTokenImpl<LexTokenKind::SharedKeyword>(TextSpan(start, _position));
case Hash(u8"super"): return new LexTokenImpl<LexTokenKind::SuperKeyword>(TextSpan(start, _position));
case Hash(u8"switch"): return new LexTokenImpl<LexTokenKind::SwitchKeyword>(TextSpan(start, _position));
case Hash(u8"this"): return new LexTokenImpl<LexTokenKind::ThisKeyword>(TextSpan(start, _position));
case Hash(u8"true"): return new LexTokenImpl<LexTokenKind::TrueKeyword>(TextSpan(start, _position));
case Hash(u8"try"): return new LexTokenImpl<LexTokenKind::TryKeyword>(TextSpan(start, _position));
case Hash(u8"typedef"): return new LexTokenImpl<LexTokenKind::TypedefKeyword>(TextSpan(start, _position));
case Hash(u8"uint"): return new LexTokenImpl<LexTokenKind::UintKeyword>(TextSpan(start, _position));
case Hash(u8"uint8"): return new LexTokenImpl<LexTokenKind::Uint8Keyword>(TextSpan(start, _position));
case Hash(u8"uint16"): return new LexTokenImpl<LexTokenKind::Uint16Keyword>(TextSpan(start, _position));
case Hash(u8"uint32"): return new LexTokenImpl<LexTokenKind::Uint32Keyword>(TextSpan(start, _position));
case Hash(u8"uint64"): return new LexTokenImpl<LexTokenKind::Uint64Keyword>(TextSpan(start, _position));
case Hash(u8"void"): return new LexTokenImpl<LexTokenKind::VoidKeyword>(TextSpan(start, _position));
case Hash(u8"while"): return new LexTokenImpl<LexTokenKind::WhileKeyword>(TextSpan(start, _position));
case Hash(u8"xor"): return new LexTokenImpl<LexTokenKind::XorKeyword>(TextSpan(start, _position));
return Create<LexTokenImpl<LexTokenKind::ProtectedKeyword>>(TextSpan(start, _position));
case Hash(u8"return"): return Create<LexTokenImpl<LexTokenKind::ReturnKeyword>>(TextSpan(start, _position));
case Hash(u8"set"): return Create<LexTokenImpl<LexTokenKind::SetKeyword>>(TextSpan(start, _position));
case Hash(u8"shared"): return Create<LexTokenImpl<LexTokenKind::SharedKeyword>>(TextSpan(start, _position));
case Hash(u8"super"): return Create<LexTokenImpl<LexTokenKind::SuperKeyword>>(TextSpan(start, _position));
case Hash(u8"switch"): return Create<LexTokenImpl<LexTokenKind::SwitchKeyword>>(TextSpan(start, _position));
case Hash(u8"this"): return Create<LexTokenImpl<LexTokenKind::ThisKeyword>>(TextSpan(start, _position));
case Hash(u8"true"): return Create<LexTokenImpl<LexTokenKind::TrueKeyword>>(TextSpan(start, _position));
case Hash(u8"try"): return Create<LexTokenImpl<LexTokenKind::TryKeyword>>(TextSpan(start, _position));
case Hash(u8"typedef"): return Create<LexTokenImpl<LexTokenKind::TypedefKeyword>>(TextSpan(start, _position));
case Hash(u8"uint"): return Create<LexTokenImpl<LexTokenKind::UintKeyword>>(TextSpan(start, _position));
case Hash(u8"uint8"): return Create<LexTokenImpl<LexTokenKind::Uint8Keyword>>(TextSpan(start, _position));
case Hash(u8"uint16"): return Create<LexTokenImpl<LexTokenKind::Uint16Keyword>>(TextSpan(start, _position));
case Hash(u8"uint32"): return Create<LexTokenImpl<LexTokenKind::Uint32Keyword>>(TextSpan(start, _position));
case Hash(u8"uint64"): return Create<LexTokenImpl<LexTokenKind::Uint64Keyword>>(TextSpan(start, _position));
case Hash(u8"void"): return Create<LexTokenImpl<LexTokenKind::VoidKeyword>>(TextSpan(start, _position));
case Hash(u8"while"): return Create<LexTokenImpl<LexTokenKind::WhileKeyword>>(TextSpan(start, _position));
case Hash(u8"xor"): return Create<LexTokenImpl<LexTokenKind::XorKeyword>>(TextSpan(start, _position));
default: return new IdentifierToken(TextSpan(start, _position), str);
default: return Create<IdentifierToken>(TextSpan(start, _position), str);
}
}
bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) {

19
src/Parser/Lexer/Lexer.hpp
View File

@ -3,6 +3,7 @@
#include <string_view>
#include "../../Diagnostics/Diagnostics.hpp"
#include "../../Utils/MemoryAllocator.hpp"
#include "LexToken.hpp"
namespace ElohimScript::Parser {
@ -10,17 +11,21 @@ namespace ElohimScript::Parser {
public:
Lexer(const char* script, Diagnostics::Diagnostics* diag)
: Lexer(reinterpret_cast<const char8_t*>(script), diag) {}
Lexer(const char8_t* script, Diagnostics::Diagnostics* diag) : _script(script), _diagnostics(diag) {}
Lexer(std::u8string_view script, Diagnostics::Diagnostics* diag) : _script(script), _diagnostics(diag) {}
Lexer(const char8_t* script, Diagnostics::Diagnostics* diag) : Lexer(std::u8string_view(script), diag) {}
Lexer(std::u8string_view script, Diagnostics::Diagnostics* diag)
: _script(script), _scriptLength(script.size()), _diagnostics(diag) {}
const LexToken* Lex();
private:
std::u8string_view _script;
size_t _position = -1;
size_t _scriptLength;
Diagnostics::Diagnostics* _diagnostics;
Utils::MemoryAllocator<2048, 1024> _allocator;
inline char8_t Consume() {
if (++_position >= _script.size()) {
if (++_position >= _scriptLength) {
return '\0';
}
return _script[_position];
@ -30,10 +35,10 @@ namespace ElohimScript::Parser {
inline char8_t Peek(size_t offset = 1) {
auto pos = _position + offset;
if (pos >= _script.size()) {
if (pos >= _scriptLength) {
return '\0';
}
return _script[pos];
return *(_script.data() + pos);
}
LexToken* LexNext();
@ -47,6 +52,10 @@ namespace ElohimScript::Parser {
LexToken* LexKeywordOrIdentifier();
static bool IsAlphaNumericalOrUnderscore(char8_t c);
template <class T, class... parameters> inline T* Create(parameters... args) {
return _allocator.Create<T>(args...);
}
};
}

45
src/Utils/MemoryAllocator.hpp Normal file
View File

@ -0,0 +1,45 @@
#ifndef ELOHIMSCRIPT_MEMORYALLOCATOR_HPP
#define ELOHIMSCRIPT_MEMORYALLOCATOR_HPP
#include <cstddef>
#include <cstdint>
#include <stdexcept>
namespace ElohimScript::Utils {
template <size_t initialSize, size_t steps>
class MemoryAllocator {
size_t _offset;
size_t _capacity;
uint8_t* _memory = nullptr;
public:
MemoryAllocator() : _offset(0), _capacity(initialSize) {
auto *ptr = malloc(_capacity);
if (ptr == nullptr) {
throw std::logic_error("Out of memory.");
}
_memory = static_cast<uint8_t*>(ptr);
}
~MemoryAllocator(){
free(_memory);
}
template <class T, class... parameters> T* Create(parameters... args){
if (_offset + sizeof(T) >= _capacity) {
_capacity += steps;
auto newPtr = realloc(_memory, _capacity);
if (newPtr == nullptr) {
throw std::logic_error("Out of memory.");
}
_memory = static_cast<uint8_t*>(newPtr);
}
uint8_t* ptr = _memory + _offset;
T* element = new (ptr) T(args...);
_offset += sizeof(T);
return element;
}
};
}
#endif // ELOHIMSCRIPT_MEMORYALLOCATOR_HPP

2
tests/LexerTests/IdentifierLexTests.cpp
View File

@ -11,7 +11,6 @@ using namespace ElohimScript::Parser;
CHECK(diag.GetMessages().empty()); \
CHECK(token->GetKind() == LexTokenKind::symbol); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
KEYWORD_TEST("and", AndKeyword);
@ -101,7 +100,6 @@ namespace doctest {
auto value = ((IdentifierToken*)token)->GetValue(); \
CHECK(value == std::u8string(reinterpret_cast<const char8_t*>(identifier))); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
IDENTIFIER_TEST("foobar");

1
tests/LexerTests/LexerIntegrationTests.cpp
View File

@ -17,7 +17,6 @@ using namespace ElohimScript::Parser;
pos++; \
current = current->GetNext().get(); \
} \
delete token; \
}
LEX_TEST("1 + 1", LexTokenKind::IntegerLiteral, LexTokenKind::Whitespace, LexTokenKind::PlusSymbol,

5
tests/LexerTests/NumericalLexTests.cpp
View File

@ -13,7 +13,6 @@ using namespace ElohimScript::Parser;
auto value = ((const IntegerLiteral*)token)->GetValue(); \
CHECK(value == (expected)); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
#define FLOAT_TEST(script, expected) \
@ -26,7 +25,6 @@ using namespace ElohimScript::Parser;
auto value = ((const FloatLiteral*)token)->GetValue(); \
CHECK(value == (expected)); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
// Decimal lexing
@ -72,11 +70,10 @@ INTEGER_TEST("0b110011", 51);
TEST_CASE("Lex invalid numerical base") {
ElohimScript::Diagnostics::Diagnostics diag;
auto lexer = Lexer("0f553", &diag);
const auto* token = lexer.Lex();
lexer.Lex();
const auto& messages = diag.GetMessages();
REQUIRE(messages.size() == 1);
CHECK(messages[0].GetType() == ElohimScript::Diagnostics::DiagnosticType::InvalidNumericalBase);
CHECK(messages[0].GetLevel() == ElohimScript::Diagnostics::DiagnosticLevel::Error);
CHECK(messages[0].GetSpan() == ElohimScript::TextSpan(0, 2));
delete token;
}

2
tests/LexerTests/StringLexTests.cpp
View File

@ -13,7 +13,6 @@ using namespace ElohimScript::Parser;
auto value = ((const StringLiteral*)token)->GetValue(); \
CHECK(value == std::u8string(reinterpret_cast<const char8_t*>(str))); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
STRING_TEST("foo bar", "'");
@ -34,5 +33,4 @@ bar""")",
CHECK(value == std::u8string(reinterpret_cast<const char8_t*>(R"(foo
bar)")));
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile);
delete token;
}

2
tests/LexerTests/SymbolLexTests.cpp
View File

@ -12,7 +12,6 @@ using namespace ElohimScript::Parser;
CHECK(diag.GetMessages().empty()); \
CHECK(token->GetKind() == LexTokenKind::symbol); \
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile); \
delete token; \
}
SYMBOL_TEST("*", StarSymbol)
@ -80,6 +79,5 @@ TEST_CASE("Lex whitespace") {
CHECK(diag.GetMessages().empty());
CHECK(token->GetKind() == LexTokenKind::Whitespace);
CHECK(token->GetNext()->GetKind() == LexTokenKind::EndOfFile);
delete token;
}
}