Deukhoofd
/
MalachScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes issue in the lexer where most tokens had the wrong textspan.

This commit is contained in:
Deukhoofd 2021-01-02 19:08:46 +01:00
parent 9109b7513a
commit ee53e7abd8
Signed by: Deukhoofd
GPG Key ID: F63E044490819F6F
1 changed files with 49 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
src/Parser/Lexer/Lexer.cpp
View File

@ -25,7 +25,7 @@ namespace MalachScript::Parser {
auto start = _position;
auto c = Consume();
switch (c) {
case u8'\0': return Create<LexTokenImpl<LexTokenKind::EndOfFile>>(TextSpan(start, 1));
case u8'\0': return Create<LexTokenImpl<LexTokenKind::EndOfFile>>(TextSpan(start, start + 1));
case u8'*': {
auto n = Peek();
if (n == u8'*') {
@ -34,91 +34,92 @@ namespace MalachScript::Parser {
if (n == u8'=') {
Progress();
// **=
return Create<LexTokenImpl<LexTokenKind::StarStarEqualsSymbol>>(TextSpan(start, 3));
return Create<LexTokenImpl<LexTokenKind::StarStarEqualsSymbol>>(TextSpan(start, start + 3));
}
// **
return Create<LexTokenImpl<LexTokenKind::StarStarSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::StarStarSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'=') {
Progress();
// *=
return Create<LexTokenImpl<LexTokenKind::StarEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::StarEqualsSymbol>>(TextSpan(start, start + 2));
}
// *
return Create<LexTokenImpl<LexTokenKind::StarSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::StarSymbol>>(TextSpan(start, start + 1));
}
case u8'/':
if (Peek() == u8'=') {
Progress();
// /=
return Create<LexTokenImpl<LexTokenKind::SlashEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::SlashEqualsSymbol>>(TextSpan(start, start + 2));
}
// /
return Create<LexTokenImpl<LexTokenKind::SlashSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::SlashSymbol>>(TextSpan(start, start + 1));
case u8'%':
if (Peek() == u8'=') {
Progress();
// %=
return Create<LexTokenImpl<LexTokenKind::PercentEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::PercentEqualsSymbol>>(TextSpan(start, start + 2));
}
// %
return Create<LexTokenImpl<LexTokenKind::PercentSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::PercentSymbol>>(TextSpan(start, start + 1));
case u8'+': {
auto n = Peek();
if (n == u8'=') {
Progress();
// +=
return Create<LexTokenImpl<LexTokenKind::PlusEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::PlusEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'+') {
Progress();
// ++
return Create<LexTokenImpl<LexTokenKind::PlusPlusSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::PlusPlusSymbol>>(TextSpan(start, start + 2));
}
// +
return Create<LexTokenImpl<LexTokenKind::PlusSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::PlusSymbol>>(TextSpan(start, start + 1));
}
case u8'-': {
auto n = Peek();
if (n == u8'=') {
Progress();
// -=
return Create<LexTokenImpl<LexTokenKind::MinusEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::MinusEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'-') {
Progress();
// --
return Create<LexTokenImpl<LexTokenKind::MinusMinusSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::MinusMinusSymbol>>(TextSpan(start, start + 2));
}
// -
return Create<LexTokenImpl<LexTokenKind::MinusSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::MinusSymbol>>(TextSpan(start, start + 1));
}
case u8'<': {
auto n = Peek();
if (n == u8'=') {
Progress();
// <=
return Create<LexTokenImpl<LexTokenKind::LessThanEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::LessThanEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'<') {
Progress();
if (Peek() == u8'=') {
Progress();
// <<=
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanEqualsSymbol>>(TextSpan(start, 3));
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanEqualsSymbol>>(
TextSpan(start, start + 3));
}
// <<
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::LessThanLessThanSymbol>>(TextSpan(start, start + 2));
}
// <
return Create<LexTokenImpl<LexTokenKind::LessThanSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::LessThanSymbol>>(TextSpan(start, start + 1));
}
case u8'>': {
auto n = Peek();
if (n == u8'=') {
Progress();
// >=
return Create<LexTokenImpl<LexTokenKind::GreaterThanEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::GreaterThanEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'>') {
Progress();
@ -127,7 +128,7 @@ namespace MalachScript::Parser {
Progress();
// >>=
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanEqualsSymbol>>(
TextSpan(start, 3));
TextSpan(start, start + 3));
}
if (n == u8'>') {
Progress();
@ -135,87 +136,90 @@ namespace MalachScript::Parser {
Progress();
// >>>=
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanEqualsSymbol>>(
TextSpan(start, 4));
TextSpan(start, start + 4));
}
// >>>
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanGreaterThanSymbol>>(
TextSpan(start, 3));
TextSpan(start, start + 3));
}
// >>
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::GreaterThanGreaterThanSymbol>>(TextSpan(start, start + 2));
}
// >
return Create<LexTokenImpl<LexTokenKind::GreaterThanSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::GreaterThanSymbol>>(TextSpan(start, start + 1));
}
case u8'(': return Create<LexTokenImpl<LexTokenKind::OpenParenthesisSymbol>>(TextSpan(start, 1));
case u8')': return Create<LexTokenImpl<LexTokenKind::CloseParenthesisSymbol>>(TextSpan(start, 1));
case u8'(': return Create<LexTokenImpl<LexTokenKind::OpenParenthesisSymbol>>(TextSpan(start, start + 1));
case u8')': return Create<LexTokenImpl<LexTokenKind::CloseParenthesisSymbol>>(TextSpan(start, start + 1));
case u8'=': {
if (Peek() == u8'=') {
Progress();
// ==
return Create<LexTokenImpl<LexTokenKind::EqualsEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::EqualsEqualsSymbol>>(TextSpan(start, start + 2));
}
// =
return Create<LexTokenImpl<LexTokenKind::EqualsSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::EqualsSymbol>>(TextSpan(start, start + 1));
}
case u8'!': {
auto n = Peek();
if (n == u8'=') {
Progress();
// !=
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'i' && Peek(2) == u8's') {
Progress(2);
// !is
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkIsSymbol>>(TextSpan(start, 3));
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkIsSymbol>>(TextSpan(start, start + 3));
}
// !
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::ExclamationMarkSymbol>>(TextSpan(start, start + 1));
}
case u8'?': return Create<LexTokenImpl<LexTokenKind::QuestionMarkSymbol>>(TextSpan(start, 1));
case u8'?': return Create<LexTokenImpl<LexTokenKind::QuestionMarkSymbol>>(TextSpan(start, start + 1));
case u8':': {
if (Peek() == u8':') {
Progress();
// ::
return Create<LexTokenImpl<LexTokenKind::ColonColonSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::ColonColonSymbol>>(TextSpan(start, start + 2));
}
// :
return Create<LexTokenImpl<LexTokenKind::ColonSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::ColonSymbol>>(TextSpan(start, start + 1));
}
case u8'&': {
auto n = Peek();
if (n == u8'=') {
Progress();
// &=
return Create<LexTokenImpl<LexTokenKind::AmpersandEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::AmpersandEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'&') {
Progress();
// &&
return Create<LexTokenImpl<LexTokenKind::AmpersandAmpersandSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::AmpersandAmpersandSymbol>>(TextSpan(start, start + 2));
}
// &
return Create<LexTokenImpl<LexTokenKind::AmpersandSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::AmpersandSymbol>>(TextSpan(start, start + 1));
}
case u8',': return Create<LexTokenImpl<LexTokenKind::CommaSymbol>>(TextSpan(start, 1));
case u8'{': return Create<LexTokenImpl<LexTokenKind::OpenCurlyParenthesisSymbol>>(TextSpan(start, 1));
case u8'}': return Create<LexTokenImpl<LexTokenKind::CloseCurlyParenthesisSymbol>>(TextSpan(start, 1));
case u8';': return Create<LexTokenImpl<LexTokenKind::SemicolonSymbol>>(TextSpan(start, 1));
case u8',': return Create<LexTokenImpl<LexTokenKind::CommaSymbol>>(TextSpan(start, start + 1));
case u8'{':
return Create<LexTokenImpl<LexTokenKind::OpenCurlyParenthesisSymbol>>(TextSpan(start, start + 1));
case u8'}':
return Create<LexTokenImpl<LexTokenKind::CloseCurlyParenthesisSymbol>>(TextSpan(start, start + 1));
case u8';': return Create<LexTokenImpl<LexTokenKind::SemicolonSymbol>>(TextSpan(start, start + 1));
case u8'|': {
auto n = Peek();
if (n == u8'=') {
Progress();
// |=
return Create<LexTokenImpl<LexTokenKind::VerticalLineEqualsSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::VerticalLineEqualsSymbol>>(TextSpan(start, start + 2));
}
if (n == u8'|') {
Progress();
// ||
return Create<LexTokenImpl<LexTokenKind::VerticalLineVerticalLineSymbol>>(TextSpan(start, 2));
return Create<LexTokenImpl<LexTokenKind::VerticalLineVerticalLineSymbol>>(
TextSpan(start, start + 2));
}
// |
return Create<LexTokenImpl<LexTokenKind::VerticalLineSymbol>>(TextSpan(start, 1));
return Create<LexTokenImpl<LexTokenKind::VerticalLineSymbol>>(TextSpan(start, start + 1));
}
case u8'^': {
auto n = Peek();