718 lines
34 KiB
C++
718 lines
34 KiB
C++
#include "Lexer.hpp"
|
|
#include <cmath>
|
|
#include <stdexcept>
|
|
#include "NumericalLexers.hpp"
|
|
|
|
namespace MalachScript::Parser {
|
|
const LexToken* Lexer::Lex() {
|
|
auto* first = LexNext();
|
|
if (first->GetKind() == LexTokenKind::EndOfFile) {
|
|
return first;
|
|
}
|
|
auto* last = first;
|
|
while (true) {
|
|
auto* next = LexNext();
|
|
last->SetNext(next);
|
|
last = next;
|
|
if (next->GetKind() == LexTokenKind::EndOfFile) {
|
|
break;
|
|
}
|
|
}
|
|
return first;
|
|
}
|
|
|
|
LexToken* Lexer::LexNext() {
|
|
auto start = _position;
|
|
auto c = Consume();
|
|
switch (c) {
|
|
case '\0':
|
|
return Create<LexTokenImpl<LexTokenKind::EndOfFile>>(ScriptTextSpan(start + 1, start + 2, _scriptName));
|
|
case '*': {
|
|
auto n = Peek();
|
|
if (n == '*') {
|
|
Progress();
|
|
n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// **=
|
|
return Create<LexTokenImpl<LexTokenKind::StarStarEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 3, _scriptName));
|
|
}
|
|
// **
|
|
return Create<LexTokenImpl<LexTokenKind::StarStarSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '=') {
|
|
Progress();
|
|
// *=
|
|
return Create<LexTokenImpl<LexTokenKind::StarEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// *
|
|
return Create<LexTokenImpl<LexTokenKind::StarSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '/':
|
|
if (Peek() == '=') {
|
|
Progress();
|
|
// /=
|
|
return Create<LexTokenImpl<LexTokenKind::SlashEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
} else if (Peek() == '/') {
|
|
Progress();
|
|
while (true) {
|
|
auto next = Consume();
|
|
if (next == '\n' || next == '\0') {
|
|
return Create<LexTokenImpl<LexTokenKind::Comment>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
}
|
|
}
|
|
} else if (Peek() == '*') {
|
|
Progress();
|
|
while (true) {
|
|
auto next = Consume();
|
|
if ((next == '*' && Peek() == '/') || next == '\0') {
|
|
Progress();
|
|
return Create<LexTokenImpl<LexTokenKind::Comment>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
}
|
|
}
|
|
}
|
|
// /
|
|
return Create<LexTokenImpl<LexTokenKind::SlashSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '%':
|
|
if (Peek() == '=') {
|
|
Progress();
|
|
// %=
|
|
return Create<LexTokenImpl<LexTokenKind::PercentEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// %
|
|
return Create<LexTokenImpl<LexTokenKind::PercentSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '+': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// +=
|
|
return Create<LexTokenImpl<LexTokenKind::PlusEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '+') {
|
|
Progress();
|
|
// ++
|
|
return Create<LexTokenImpl<LexTokenKind::PlusPlusSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// +
|
|
return Create<LexTokenImpl<LexTokenKind::PlusSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '-': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// -=
|
|
return Create<LexTokenImpl<LexTokenKind::MinusEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '-') {
|
|
Progress();
|
|
// --
|
|
return Create<LexTokenImpl<LexTokenKind::MinusMinusSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// -
|
|
return Create<LexTokenImpl<LexTokenKind::MinusSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '<': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// <=
|
|
return Create<LexTokenImpl<LexTokenKind::LessThanEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '<') {
|
|
Progress();
|
|
if (Peek() == '=') {
|
|
Progress();
|
|
// <<=
|
|
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 3, _scriptName));
|
|
}
|
|
// <<
|
|
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// <
|
|
return Create<LexTokenImpl<LexTokenKind::LessThanSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '>': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// >=
|
|
return Create<LexTokenImpl<LexTokenKind::GreaterThanEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '>') {
|
|
Progress();
|
|
n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// >>=
|
|
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 3, _scriptName));
|
|
}
|
|
if (n == '>') {
|
|
Progress();
|
|
if (Peek() == '=') {
|
|
Progress();
|
|
// >>>=
|
|
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 4, _scriptName));
|
|
}
|
|
// >>>
|
|
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanSymbol>>(
|
|
ScriptTextSpan(start, start + 3, _scriptName));
|
|
}
|
|
// >>
|
|
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// >
|
|
return Create<LexTokenImpl<LexTokenKind::GreaterThanSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '(':
|
|
return Create<LexTokenImpl<LexTokenKind::OpenParenthesisSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case ')':
|
|
return Create<LexTokenImpl<LexTokenKind::CloseParenthesisSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '=': {
|
|
if (Peek() == '=') {
|
|
Progress();
|
|
// ==
|
|
return Create<LexTokenImpl<LexTokenKind::EqualsEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// =
|
|
return Create<LexTokenImpl<LexTokenKind::EqualsSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '!': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// !=
|
|
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == 'i' && Peek(2) == 's') {
|
|
Progress(2);
|
|
// !is
|
|
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkIsSymbol>>(
|
|
ScriptTextSpan(start, start + 3, _scriptName));
|
|
}
|
|
// !
|
|
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '?':
|
|
return Create<LexTokenImpl<LexTokenKind::QuestionMarkSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case ':': {
|
|
if (Peek() == ':') {
|
|
Progress();
|
|
// ::
|
|
return Create<LexTokenImpl<LexTokenKind::ColonColonSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// :
|
|
return Create<LexTokenImpl<LexTokenKind::ColonSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '&': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// &=
|
|
return Create<LexTokenImpl<LexTokenKind::AmpersandEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '&') {
|
|
Progress();
|
|
// &&
|
|
return Create<LexTokenImpl<LexTokenKind::AmpersandAmpersandSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// &
|
|
return Create<LexTokenImpl<LexTokenKind::AmpersandSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case ',':
|
|
return Create<LexTokenImpl<LexTokenKind::CommaSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '{':
|
|
return Create<LexTokenImpl<LexTokenKind::OpenCurlyParenthesisSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '}':
|
|
return Create<LexTokenImpl<LexTokenKind::CloseCurlyParenthesisSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case ';':
|
|
return Create<LexTokenImpl<LexTokenKind::SemicolonSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '|': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// |=
|
|
return Create<LexTokenImpl<LexTokenKind::VerticalLineEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '|') {
|
|
Progress();
|
|
// ||
|
|
return Create<LexTokenImpl<LexTokenKind::VerticalLineVerticalLineSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// |
|
|
return Create<LexTokenImpl<LexTokenKind::VerticalLineSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '^': {
|
|
auto n = Peek();
|
|
if (n == '=') {
|
|
Progress();
|
|
// ^=
|
|
return Create<LexTokenImpl<LexTokenKind::CaretEqualsSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
if (n == '^') {
|
|
Progress();
|
|
// ^^
|
|
return Create<LexTokenImpl<LexTokenKind::CaretCaretSymbol>>(
|
|
ScriptTextSpan(start, start + 2, _scriptName));
|
|
}
|
|
// ^
|
|
return Create<LexTokenImpl<LexTokenKind::CaretSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '~':
|
|
return Create<LexTokenImpl<LexTokenKind::TildeSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '.':
|
|
return Create<LexTokenImpl<LexTokenKind::DotSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '[':
|
|
return Create<LexTokenImpl<LexTokenKind::OpenBlockParenthesisSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case ']':
|
|
return Create<LexTokenImpl<LexTokenKind::CloseBlockParenthesisSymbol>>(
|
|
ScriptTextSpan(start, start + 1, _scriptName));
|
|
case '@':
|
|
return Create<LexTokenImpl<LexTokenKind::AtSymbol>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
|
|
case ' ':
|
|
case '\r':
|
|
case '\n':
|
|
case '\t':
|
|
return Create<LexTokenImpl<LexTokenKind::Whitespace>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
// Byte order mark
|
|
case '\xEF': {
|
|
if (Peek() == '\xBB' && Peek(2) == '\xBF') {
|
|
Progress(2);
|
|
return Create<LexTokenImpl<LexTokenKind::Whitespace>>(
|
|
ScriptTextSpan(start, start + 3, _scriptName));
|
|
}
|
|
LogError(Diagnostics::DiagnosticType::UnknownCharacter, ScriptTextSpan(start, start + 1, _scriptName),
|
|
{std::string(1, c)});
|
|
return Create<LexTokenImpl<LexTokenKind::Unknown>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
case '0':
|
|
case '1':
|
|
case '2':
|
|
case '3':
|
|
case '4':
|
|
case '5':
|
|
case '6':
|
|
case '7':
|
|
case '8':
|
|
case '9': return LexNumerical(c);
|
|
case '\'': return LexString('\'', false);
|
|
case '"': {
|
|
if (Peek() == '"' && Peek(2) == '\"') {
|
|
return LexString('"', true);
|
|
}
|
|
return LexString('"', false);
|
|
}
|
|
|
|
default:
|
|
if (IsAlphaNumericalOrUnderscore(c))
|
|
return LexKeywordOrIdentifier();
|
|
LogError(Diagnostics::DiagnosticType::UnknownCharacter, ScriptTextSpan(start, start + 1, _scriptName),
|
|
{std::string(1, (char)c)});
|
|
return Create<LexTokenImpl<LexTokenKind::Unknown>>(ScriptTextSpan(start, start + 1, _scriptName));
|
|
}
|
|
}
|
|
|
|
LexToken* Lexer::LexNumerical(char c) {
|
|
auto initialValue = LexDecimalValue(c);
|
|
auto numericalSystem = 10; // Default to decimal system.
|
|
if (initialValue == 0) {
|
|
auto secondChar = Peek();
|
|
auto secondValue = LexDecimalValue(secondChar);
|
|
if (secondChar != '.' && secondValue == 255) {
|
|
switch (secondChar) {
|
|
case 'x':
|
|
case 'X':
|
|
Progress();
|
|
numericalSystem = 16;
|
|
break;
|
|
case 'd':
|
|
case 'D':
|
|
Progress();
|
|
numericalSystem = 10;
|
|
break;
|
|
case 'o':
|
|
case 'O':
|
|
Progress();
|
|
numericalSystem = 8;
|
|
break;
|
|
case 'b':
|
|
case 'B':
|
|
Progress();
|
|
numericalSystem = 2;
|
|
break;
|
|
default: return Create<IntegerLiteral>(ScriptTextSpan(_position - 1, _position, _scriptName), 0);
|
|
}
|
|
}
|
|
}
|
|
switch (numericalSystem) {
|
|
case 10: return LexDecimal(initialValue);
|
|
case 16: return LexHexadecimal();
|
|
case 8: return LexOctal();
|
|
case 2: return LexBinary();
|
|
default: throw std::logic_error("Not implemented");
|
|
}
|
|
}
|
|
|
|
constexpr int64_t quick_pow10(int n) {
|
|
constexpr int64_t pow10[20] = {1,
|
|
10,
|
|
100,
|
|
1000,
|
|
10000,
|
|
100000,
|
|
1000000,
|
|
10000000,
|
|
100000000,
|
|
1000000000,
|
|
10000000000,
|
|
100000000000,
|
|
1000000000000,
|
|
10000000000000,
|
|
10000000000000,
|
|
100000000000000,
|
|
1000000000000000,
|
|
10000000000000000,
|
|
100000000000000000,
|
|
1000000000000000000};
|
|
return pow10[n];
|
|
}
|
|
|
|
LexToken* Lexer::LexDecimal(ParseInt initial) {
|
|
auto start = _position;
|
|
ParseInt value = initial;
|
|
ParseInt decimalValue = 0;
|
|
ParseInt exponentValue = 0;
|
|
uint8_t decimalLength = 0;
|
|
bool isDecimal = false;
|
|
bool isExponent = false;
|
|
while (true) {
|
|
auto v = (ParseInt)LexDecimalValue(Peek());
|
|
if (v == 255) {
|
|
if (!isDecimal && Peek() == '.') {
|
|
isDecimal = true;
|
|
Progress();
|
|
continue;
|
|
}
|
|
if (isDecimal && (Peek() == 'e' || Peek() == 'E')) {
|
|
isDecimal = false;
|
|
isExponent = true;
|
|
Progress();
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
Progress();
|
|
if (isDecimal) {
|
|
decimalValue *= 10;
|
|
decimalValue += v;
|
|
decimalLength++;
|
|
} else if (isExponent) {
|
|
exponentValue *= 10;
|
|
exponentValue += v;
|
|
} else {
|
|
value *= 10;
|
|
value += v;
|
|
}
|
|
}
|
|
if (isDecimal || isExponent) {
|
|
auto val = value + ((ParseFloat)decimalValue / quick_pow10(decimalLength));
|
|
if (isExponent) {
|
|
val *= pow(10, exponentValue);
|
|
}
|
|
return Create<FloatLiteral>(ScriptTextSpan(start, _position, _scriptName), val);
|
|
}
|
|
return Create<IntegerLiteral>(ScriptTextSpan(start, _position, _scriptName), value);
|
|
}
|
|
|
|
IntegerLiteral* Lexer::LexHexadecimal() {
|
|
auto start = _position;
|
|
ParseInt value = 0;
|
|
while (true) {
|
|
auto v = LexHexadecimalValue(Peek());
|
|
if (v == 255) {
|
|
break;
|
|
}
|
|
Progress();
|
|
value <<= 4;
|
|
value += v;
|
|
}
|
|
return Create<IntegerLiteral>(ScriptTextSpan(start - 1, _position, _scriptName), value);
|
|
}
|
|
IntegerLiteral* Lexer::LexOctal() {
|
|
auto start = _position;
|
|
ParseInt value = 0;
|
|
while (true) {
|
|
auto v = LexOctalValue(Peek());
|
|
if (v == 255) {
|
|
break;
|
|
}
|
|
Progress();
|
|
value <<= 3;
|
|
value += v;
|
|
}
|
|
return Create<IntegerLiteral>(ScriptTextSpan(start - 1, _position, _scriptName), value);
|
|
}
|
|
IntegerLiteral* Lexer::LexBinary() {
|
|
auto start = _position;
|
|
ParseInt value = 0;
|
|
while (true) {
|
|
auto v = LexBinaryValue(Peek());
|
|
if (v == 255) {
|
|
break;
|
|
}
|
|
Progress();
|
|
value <<= 1;
|
|
value += v;
|
|
}
|
|
return Create<IntegerLiteral>(ScriptTextSpan(start - 1, _position, _scriptName), value);
|
|
}
|
|
StringLiteral* Lexer::LexString(char opening, bool heredoc) {
|
|
auto openingPos = _position;
|
|
Progress();
|
|
if (heredoc) {
|
|
Progress(2);
|
|
}
|
|
auto start = _position;
|
|
size_t offset = 0;
|
|
while (true) {
|
|
auto current = Peek(offset);
|
|
if (heredoc) {
|
|
if (current == '"' && Peek(offset + 1) == '"' && Peek(offset + 2) == '"' && Peek(offset + 3) != '"') {
|
|
break;
|
|
}
|
|
} else if (current == opening) {
|
|
break;
|
|
}
|
|
if (current == '\0') {
|
|
LogError(Diagnostics::DiagnosticType::ExpectedEndOfString,
|
|
ScriptTextSpan(start, start + offset, _scriptName), {"EndOfFile"});
|
|
break;
|
|
}
|
|
if (!heredoc && (current == '\n' || current == '\r')) {
|
|
LogError(Diagnostics::DiagnosticType::ExpectedEndOfString,
|
|
ScriptTextSpan(start, start + offset, _scriptName), {"Newline"});
|
|
break;
|
|
}
|
|
offset++;
|
|
}
|
|
Progress(offset);
|
|
if (heredoc) {
|
|
Progress(2);
|
|
}
|
|
return Create<StringLiteral>(ScriptTextSpan(openingPos, openingPos + _position, _scriptName),
|
|
ParseString(_script.substr(start, offset)));
|
|
}
|
|
|
|
LexToken* Lexer::LexKeywordOrIdentifier() {
|
|
auto start = _position;
|
|
auto offset = 0;
|
|
while (IsAlphaNumericalOrUnderscore(Peek(offset))) {
|
|
offset++;
|
|
}
|
|
auto str = _script.substr(start, offset);
|
|
Progress(offset - 1);
|
|
auto hash = Identifier::Hash(str);
|
|
switch (hash) {
|
|
case Identifier::Hash("and"):
|
|
return Create<LexTokenImpl<LexTokenKind::AndKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("abstract"):
|
|
return Create<LexTokenImpl<LexTokenKind::AbstractKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("auto"):
|
|
return Create<LexTokenImpl<LexTokenKind::AutoKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("bool"):
|
|
return Create<LexTokenImpl<LexTokenKind::BoolKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("break"):
|
|
return Create<LexTokenImpl<LexTokenKind::BreakKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("case"):
|
|
return Create<LexTokenImpl<LexTokenKind::CaseKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("cast"):
|
|
return Create<LexTokenImpl<LexTokenKind::CastKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("catch"):
|
|
return Create<LexTokenImpl<LexTokenKind::CatchKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("class"):
|
|
return Create<LexTokenImpl<LexTokenKind::ClassKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("const"):
|
|
return Create<LexTokenImpl<LexTokenKind::ConstKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("continue"):
|
|
return Create<LexTokenImpl<LexTokenKind::ContinueKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("default"):
|
|
return Create<LexTokenImpl<LexTokenKind::DefaultKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("do"):
|
|
return Create<LexTokenImpl<LexTokenKind::DoKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("double"):
|
|
return Create<LexTokenImpl<LexTokenKind::DoubleKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("else"):
|
|
return Create<LexTokenImpl<LexTokenKind::ElseKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("enum"):
|
|
return Create<LexTokenImpl<LexTokenKind::EnumKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("explicit"):
|
|
return Create<LexTokenImpl<LexTokenKind::ExplicitKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("external"):
|
|
return Create<LexTokenImpl<LexTokenKind::ExternalKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("false"):
|
|
return Create<LexTokenImpl<LexTokenKind::FalseKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("final"):
|
|
return Create<LexTokenImpl<LexTokenKind::FinalKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("float"):
|
|
return Create<LexTokenImpl<LexTokenKind::FloatKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("for"):
|
|
return Create<LexTokenImpl<LexTokenKind::ForKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("from"):
|
|
return Create<LexTokenImpl<LexTokenKind::FromKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("funcdef"):
|
|
return Create<LexTokenImpl<LexTokenKind::FuncdefKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("function"):
|
|
return Create<LexTokenImpl<LexTokenKind::FunctionKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("get"):
|
|
return Create<LexTokenImpl<LexTokenKind::GetKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("if"):
|
|
return Create<LexTokenImpl<LexTokenKind::IfKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("import"):
|
|
return Create<LexTokenImpl<LexTokenKind::ImportKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("in"):
|
|
return Create<LexTokenImpl<LexTokenKind::InKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("inout"):
|
|
return Create<LexTokenImpl<LexTokenKind::InoutKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("int"):
|
|
return Create<LexTokenImpl<LexTokenKind::IntKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("interface"):
|
|
return Create<LexTokenImpl<LexTokenKind::InterfaceKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("int8"):
|
|
return Create<LexTokenImpl<LexTokenKind::Int8Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("int16"):
|
|
return Create<LexTokenImpl<LexTokenKind::Int16Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("int32"):
|
|
return Create<LexTokenImpl<LexTokenKind::Int32Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("int64"):
|
|
return Create<LexTokenImpl<LexTokenKind::Int64Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("is"):
|
|
return Create<LexTokenImpl<LexTokenKind::IsKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("mixin"):
|
|
return Create<LexTokenImpl<LexTokenKind::MixinKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("namespace"):
|
|
return Create<LexTokenImpl<LexTokenKind::NamespaceKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("not"):
|
|
return Create<LexTokenImpl<LexTokenKind::NotKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("null"):
|
|
return Create<LexTokenImpl<LexTokenKind::NullKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("or"):
|
|
return Create<LexTokenImpl<LexTokenKind::OrKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("out"):
|
|
return Create<LexTokenImpl<LexTokenKind::OutKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("override"):
|
|
return Create<LexTokenImpl<LexTokenKind::OverrideKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("private"):
|
|
return Create<LexTokenImpl<LexTokenKind::PrivateKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("property"):
|
|
return Create<LexTokenImpl<LexTokenKind::PropertyKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("protected"):
|
|
return Create<LexTokenImpl<LexTokenKind::ProtectedKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("return"):
|
|
return Create<LexTokenImpl<LexTokenKind::ReturnKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("set"):
|
|
return Create<LexTokenImpl<LexTokenKind::SetKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("shared"):
|
|
return Create<LexTokenImpl<LexTokenKind::SharedKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("super"):
|
|
return Create<LexTokenImpl<LexTokenKind::SuperKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("switch"):
|
|
return Create<LexTokenImpl<LexTokenKind::SwitchKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("this"):
|
|
return Create<LexTokenImpl<LexTokenKind::ThisKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("true"):
|
|
return Create<LexTokenImpl<LexTokenKind::TrueKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("try"):
|
|
return Create<LexTokenImpl<LexTokenKind::TryKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("typedef"):
|
|
return Create<LexTokenImpl<LexTokenKind::TypedefKeyword>>(
|
|
ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("uint"):
|
|
return Create<LexTokenImpl<LexTokenKind::UintKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("uint8"):
|
|
return Create<LexTokenImpl<LexTokenKind::Uint8Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("uint16"):
|
|
return Create<LexTokenImpl<LexTokenKind::Uint16Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("uint32"):
|
|
return Create<LexTokenImpl<LexTokenKind::Uint32Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("uint64"):
|
|
return Create<LexTokenImpl<LexTokenKind::Uint64Keyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("void"):
|
|
return Create<LexTokenImpl<LexTokenKind::VoidKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("while"):
|
|
return Create<LexTokenImpl<LexTokenKind::WhileKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
case Identifier::Hash("xor"):
|
|
return Create<LexTokenImpl<LexTokenKind::XorKeyword>>(ScriptTextSpan(start, _position, _scriptName));
|
|
|
|
default:
|
|
return Create<IdentifierToken>(ScriptTextSpan(start, _position, _scriptName),
|
|
Identifier(str.data(), offset, hash));
|
|
}
|
|
}
|
|
bool Lexer::IsAlphaNumericalOrUnderscore(char c) {
|
|
if (c >= 'a' && c <= 'z') {
|
|
return true;
|
|
}
|
|
if (c >= 'A' && c <= 'Z') {
|
|
return true;
|
|
}
|
|
if (c >= '0' && c <= '9') {
|
|
return true;
|
|
}
|
|
if (c == '_') {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
}
|