Deukhoofd
/
MalachScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes issue in the lexer where most tokens had the wrong textspan.

This commit is contained in:
Deukhoofd 2021-01-02 19:08:46 +01:00
parent 9109b7513a
commit ee53e7abd8
Signed by: Deukhoofd
GPG Key ID: F63E044490819F6F
1 changed files with 49 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
src/Parser/Lexer/Lexer.cpp
View File

@ -25,7 +25,7 @@ namespace MalachScript::Parser {
auto start = _position; auto start = _position;
auto c = Consume(); auto c = Consume();
switch (c) { switch (c) {
case u8'\0': return Create<LexTokenImpl<LexTokenKind::EndOfFile>>(TextSpan(start, 1)); case u8'\0': return Create<LexTokenImpl<LexTokenKind::EndOfFile>>(TextSpan(start, start + 1));
case u8'*': { case u8'*': {
auto n = Peek(); auto n = Peek();
if (n == u8'*') { if (n == u8'*') {
@ -34,91 +34,92 @@ namespace MalachScript::Parser {
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// **= // **=
return Create<LexTokenImpl<LexTokenKind::StarStarEqualsSymbol>>(TextSpan(start, 3)); return Create<LexTokenImpl<LexTokenKind::StarStarEqualsSymbol>>(TextSpan(start, start + 3));
} }
// ** // **
return Create<LexTokenImpl<LexTokenKind::StarStarSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::StarStarSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// *= // *=
return Create<LexTokenImpl<LexTokenKind::StarEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::StarEqualsSymbol>>(TextSpan(start, start + 2));
} }
// * // *
return Create<LexTokenImpl<LexTokenKind::StarSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::StarSymbol>>(TextSpan(start, start + 1));
} }
case u8'/': case u8'/':
if (Peek() == u8'=') { if (Peek() == u8'=') {
Progress(); Progress();
// /= // /=
return Create<LexTokenImpl<LexTokenKind::SlashEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::SlashEqualsSymbol>>(TextSpan(start, start + 2));
} }
// / // /
return Create<LexTokenImpl<LexTokenKind::SlashSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::SlashSymbol>>(TextSpan(start, start + 1));
case u8'%': case u8'%':
if (Peek() == u8'=') { if (Peek() == u8'=') {
Progress(); Progress();
// %= // %=
return Create<LexTokenImpl<LexTokenKind::PercentEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::PercentEqualsSymbol>>(TextSpan(start, start + 2));
} }
// % // %
return Create<LexTokenImpl<LexTokenKind::PercentSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::PercentSymbol>>(TextSpan(start, start + 1));
case u8'+': { case u8'+': {
auto n = Peek(); auto n = Peek();
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// += // +=
return Create<LexTokenImpl<LexTokenKind::PlusEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::PlusEqualsSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'+') { if (n == u8'+') {
Progress(); Progress();
// ++ // ++
return Create<LexTokenImpl<LexTokenKind::PlusPlusSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::PlusPlusSymbol>>(TextSpan(start, start + 2));
} }
// + // +
return Create<LexTokenImpl<LexTokenKind::PlusSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::PlusSymbol>>(TextSpan(start, start + 1));
} }
case u8'-': { case u8'-': {
auto n = Peek(); auto n = Peek();
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// -= // -=
return Create<LexTokenImpl<LexTokenKind::MinusEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::MinusEqualsSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'-') { if (n == u8'-') {
Progress(); Progress();
// -- // --
return Create<LexTokenImpl<LexTokenKind::MinusMinusSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::MinusMinusSymbol>>(TextSpan(start, start + 2));
} }
// - // -
return Create<LexTokenImpl<LexTokenKind::MinusSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::MinusSymbol>>(TextSpan(start, start + 1));
} }
case u8'<': { case u8'<': {
auto n = Peek(); auto n = Peek();
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// <= // <=
return Create<LexTokenImpl<LexTokenKind::LessThanEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::LessThanEqualsSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'<') { if (n == u8'<') {
Progress(); Progress();
if (Peek() == u8'=') { if (Peek() == u8'=') {
Progress(); Progress();
// <<= // <<=
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanEqualsSymbol>>(TextSpan(start, 3)); return Create<LexTokenImpl<LexTokenKind::LessThanLessThanEqualsSymbol>>(
TextSpan(start, start + 3));
} }
// << // <<
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::LessThanLessThanSymbol>>(TextSpan(start, start + 2));
} }
// < // <
return Create<LexTokenImpl<LexTokenKind::LessThanSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::LessThanSymbol>>(TextSpan(start, start + 1));
} }
case u8'>': { case u8'>': {
auto n = Peek(); auto n = Peek();
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// >= // >=
return Create<LexTokenImpl<LexTokenKind::GreaterThanEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::GreaterThanEqualsSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'>') { if (n == u8'>') {
Progress(); Progress();
@ -127,7 +128,7 @@ namespace MalachScript::Parser {
Progress(); Progress();
// >>= // >>=
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanEqualsSymbol>>( return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanEqualsSymbol>>(
TextSpan(start, 3)); TextSpan(start, start + 3));
} }
if (n == u8'>') { if (n == u8'>') {
Progress(); Progress();
@ -135,87 +136,90 @@ namespace MalachScript::Parser {
Progress(); Progress();
// >>>= // >>>=
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanEqualsSymbol>>( return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanEqualsSymbol>>(
TextSpan(start, 4)); TextSpan(start, start + 4));
} }
// >>> // >>>
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanSymbol>>( return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanSymbol>>(
TextSpan(start, 3)); TextSpan(start, start + 3));
} }
// >> // >>
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanSymbol>>(TextSpan(start, start + 2));
} }
// > // >
return Create<LexTokenImpl<LexTokenKind::GreaterThanSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::GreaterThanSymbol>>(TextSpan(start, start + 1));
} }
case u8'(': return Create<LexTokenImpl<LexTokenKind::OpenParenthesisSymbol>>(TextSpan(start, 1)); case u8'(': return Create<LexTokenImpl<LexTokenKind::OpenParenthesisSymbol>>(TextSpan(start, start + 1));
case u8')': return Create<LexTokenImpl<LexTokenKind::CloseParenthesisSymbol>>(TextSpan(start, 1)); case u8')': return Create<LexTokenImpl<LexTokenKind::CloseParenthesisSymbol>>(TextSpan(start, start + 1));
case u8'=': { case u8'=': {
if (Peek() == u8'=') { if (Peek() == u8'=') {
Progress(); Progress();
// == // ==
return Create<LexTokenImpl<LexTokenKind::EqualsEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::EqualsEqualsSymbol>>(TextSpan(start, start + 2));
} }
// = // =
return Create<LexTokenImpl<LexTokenKind::EqualsSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::EqualsSymbol>>(TextSpan(start, start + 1));
} }
case u8'!': { case u8'!': {
auto n = Peek(); auto n = Peek();
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// != // !=
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::ExclamationMarkEqualsSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'i' && Peek(2) == u8's') { if (n == u8'i' && Peek(2) == u8's') {
Progress(2); Progress(2);
// !is // !is
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkIsSymbol>>(TextSpan(start, 3)); return Create<LexTokenImpl<LexTokenKind::ExclamationMarkIsSymbol>>(TextSpan(start, start + 3));
} }
// ! // !
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::ExclamationMarkSymbol>>(TextSpan(start, start + 1));
} }
case u8'?': return Create<LexTokenImpl<LexTokenKind::QuestionMarkSymbol>>(TextSpan(start, 1)); case u8'?': return Create<LexTokenImpl<LexTokenKind::QuestionMarkSymbol>>(TextSpan(start, start + 1));
case u8':': { case u8':': {
if (Peek() == u8':') { if (Peek() == u8':') {
Progress(); Progress();
// :: // ::
return Create<LexTokenImpl<LexTokenKind::ColonColonSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::ColonColonSymbol>>(TextSpan(start, start + 2));
} }
// : // :
return Create<LexTokenImpl<LexTokenKind::ColonSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::ColonSymbol>>(TextSpan(start, start + 1));
} }
case u8'&': { case u8'&': {
auto n = Peek(); auto n = Peek();
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// &= // &=
return Create<LexTokenImpl<LexTokenKind::AmpersandEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::AmpersandEqualsSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'&') { if (n == u8'&') {
Progress(); Progress();
// && // &&
return Create<LexTokenImpl<LexTokenKind::AmpersandAmpersandSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::AmpersandAmpersandSymbol>>(TextSpan(start, start + 2));
} }
// & // &
return Create<LexTokenImpl<LexTokenKind::AmpersandSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::AmpersandSymbol>>(TextSpan(start, start + 1));
} }
case u8',': return Create<LexTokenImpl<LexTokenKind::CommaSymbol>>(TextSpan(start, 1)); case u8',': return Create<LexTokenImpl<LexTokenKind::CommaSymbol>>(TextSpan(start, start + 1));
case u8'{': return Create<LexTokenImpl<LexTokenKind::OpenCurlyParenthesisSymbol>>(TextSpan(start, 1)); case u8'{':
case u8'}': return Create<LexTokenImpl<LexTokenKind::CloseCurlyParenthesisSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::OpenCurlyParenthesisSymbol>>(TextSpan(start, start + 1));
case u8';': return Create<LexTokenImpl<LexTokenKind::SemicolonSymbol>>(TextSpan(start, 1)); case u8'}':
return Create<LexTokenImpl<LexTokenKind::CloseCurlyParenthesisSymbol>>(TextSpan(start, start + 1));
case u8';': return Create<LexTokenImpl<LexTokenKind::SemicolonSymbol>>(TextSpan(start, start + 1));
case u8'|': { case u8'|': {
auto n = Peek(); auto n = Peek();
if (n == u8'=') { if (n == u8'=') {
Progress(); Progress();
// |= // |=
return Create<LexTokenImpl<LexTokenKind::VerticalLineEqualsSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::VerticalLineEqualsSymbol>>(TextSpan(start, start + 2));
} }
if (n == u8'|') { if (n == u8'|') {
Progress(); Progress();
// || // ||
return Create<LexTokenImpl<LexTokenKind::VerticalLineVerticalLineSymbol>>(TextSpan(start, 2)); return Create<LexTokenImpl<LexTokenKind::VerticalLineVerticalLineSymbol>>(
TextSpan(start, start + 2));
} }
// | // |
return Create<LexTokenImpl<LexTokenKind::VerticalLineSymbol>>(TextSpan(start, 1)); return Create<LexTokenImpl<LexTokenKind::VerticalLineSymbol>>(TextSpan(start, start + 1));
} }
case u8'^': { case u8'^': {
auto n = Peek(); auto n = Peek();