Deukhoofd/PorygonLang
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Save position and length of tokens

Browse Source
This commit is contained in:
Deukhoofd
2019-05-19 16:11:16 +02:00
parent 608a579f19
commit 7edd51d5e3
3 changed files with 84 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/Parser/Lexer.cpp
View File

@@ -34,23 +34,23 @@ char Lexer::Next(){
IToken* Lexer::LexNext(char c){
switch (c) {
case '\0':
return new SimpleToken(TokenKind::EndOfFile);
return new SimpleToken(TokenKind::EndOfFile, Lexer::Position - 1, 1);
case ' ': case '\t': case '\n': case '\r': case '\v': case '\f':
return new SimpleToken(TokenKind::WhiteSpace);
return new SimpleToken(TokenKind::WhiteSpace, Lexer::Position - 1, 1);
case '+':
return new SimpleToken(TokenKind::PlusToken);
return new SimpleToken(TokenKind::PlusToken, Lexer::Position - 1, 1);
case '-':
return new SimpleToken(TokenKind::MinusToken);
return new SimpleToken(TokenKind::MinusToken, Lexer::Position - 1, 1);
case '/':
return new SimpleToken(TokenKind::SlashToken);
return new SimpleToken(TokenKind::SlashToken, Lexer::Position - 1, 1);
case '*':
return new SimpleToken(TokenKind::StarToken);
return new SimpleToken(TokenKind::StarToken, Lexer::Position - 1, 1);
case '=':
if (Lexer::Peek() == '='){
Lexer::Next();
return new SimpleToken(TokenKind::EqualityToken);
return new SimpleToken(TokenKind::EqualityToken, Lexer::Position - 2, 2);
}
return new SimpleToken(TokenKind::AssignmentToken);
return new SimpleToken(TokenKind::AssignmentToken, Lexer::Position - 1, 1);
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
return LexNumber(c);
case '_':
@@ -85,6 +85,8 @@ IToken* Lexer::LexNumber(char c){
short decimal_index = 0;
bool has_point = false;
bool is_searching = true;
unsigned int start = Lexer::Position - 1;
unsigned int length = 1;
while (is_searching){
char next = Lexer::Peek();
int next_val = CharToInt(next);
@@ -92,12 +94,14 @@ IToken* Lexer::LexNumber(char c){
switch (next){
case '_':
Lexer::Next();
length++;
continue;
case '.':
Lexer::Next();
has_point = true;
decimal_index = 0;
float_value = int_value;
length++;
continue;
default:
is_searching = false;
@@ -106,6 +110,7 @@ IToken* Lexer::LexNumber(char c){
}
else{
Lexer::Next();
length++;
if (has_point){
decimal_index++;
float_value += next_val / pow(10, decimal_index);
@@ -117,10 +122,10 @@ IToken* Lexer::LexNumber(char c){
}
}
if (has_point){
return new FloatToken(float_value);
return new FloatToken(float_value, start, length);
}
else{
return new IntegerToken(int_value);
return new IntegerToken(int_value, start, length);
}
}
@@ -132,6 +137,7 @@ unsigned constexpr const_hash(char const *input) {
IToken* Lexer::LexIdentifierOrKeyword(char c){
vector<char> charVec(1, c);
auto start = Lexer::Position - 1;
while (true){
char next = Lexer::Peek();
if (next == '\0') break;
@@ -145,25 +151,25 @@ IToken* Lexer::LexIdentifierOrKeyword(char c){
}
string s = string(charVec.begin(), charVec.end());
switch (const_hash(s.c_str())){
case const_hash("and"): return new SimpleToken(TokenKind::AndKeyword);
case const_hash("break"): return new SimpleToken(TokenKind::BreakKeyword);
case const_hash("do"): return new SimpleToken(TokenKind::DoKeyword);
case const_hash("else"): return new SimpleToken(TokenKind::ElseKeyword);
case const_hash("elseif"): return new SimpleToken(TokenKind::ElseIfKeyword);
case const_hash("end"): return new SimpleToken(TokenKind::EndKeyword);
case const_hash("false"): return new SimpleToken(TokenKind::FalseKeyword);
case const_hash("for"): return new SimpleToken(TokenKind::ForKeyword);
case const_hash("function"): return new SimpleToken(TokenKind::FunctionKeyword);
case const_hash("if"): return new SimpleToken(TokenKind::IfKeyword);
case const_hash("in"): return new SimpleToken(TokenKind::InKeyword);
case const_hash("local"): return new SimpleToken(TokenKind::LocalKeyword);
case const_hash("nil"): return new SimpleToken(TokenKind::NilKeyword);
case const_hash("not"): return new SimpleToken(TokenKind::NotKeyword);
case const_hash("or"): return new SimpleToken(TokenKind::OrKeyword);
case const_hash("return"): return new SimpleToken(TokenKind::ReturnKeyword);
case const_hash("then"): return new SimpleToken(TokenKind::ThenKeyword);
case const_hash("true"): return new SimpleToken(TokenKind::TrueKeyword);
case const_hash("while"): return new SimpleToken(TokenKind::WhileKeyword);
default: return new IdentifierToken(s);
case const_hash("and"): return new SimpleToken(TokenKind::AndKeyword, start, 3);
case const_hash("break"): return new SimpleToken(TokenKind::BreakKeyword, start, 5);
case const_hash("do"): return new SimpleToken(TokenKind::DoKeyword, start, 2);
case const_hash("else"): return new SimpleToken(TokenKind::ElseKeyword, start, 4);
case const_hash("elseif"): return new SimpleToken(TokenKind::ElseIfKeyword, start, 6);
case const_hash("end"): return new SimpleToken(TokenKind::EndKeyword, start, 3);
case const_hash("false"): return new SimpleToken(TokenKind::FalseKeyword, start, 5);
case const_hash("for"): return new SimpleToken(TokenKind::ForKeyword, start, 3);
case const_hash("function"): return new SimpleToken(TokenKind::FunctionKeyword, start, 8);
case const_hash("if"): return new SimpleToken(TokenKind::IfKeyword, start, 2);
case const_hash("in"): return new SimpleToken(TokenKind::InKeyword, start, 2);
case const_hash("local"): return new SimpleToken(TokenKind::LocalKeyword, start, 5);
case const_hash("nil"): return new SimpleToken(TokenKind::NilKeyword, start, 3);
case const_hash("not"): return new SimpleToken(TokenKind::NotKeyword, start, 3);
case const_hash("or"): return new SimpleToken(TokenKind::OrKeyword, start, 2);
case const_hash("return"): return new SimpleToken(TokenKind::ReturnKeyword, start, 6);
case const_hash("then"): return new SimpleToken(TokenKind::ThenKeyword, start, 4);
case const_hash("true"): return new SimpleToken(TokenKind::TrueKeyword, start, 4);
case const_hash("while"): return new SimpleToken(TokenKind::WhileKeyword, start, 5);
default: return new IdentifierToken(s, start, s.length());
}
}