Deukhoofd/MalachScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implements string lexing.

Browse Source
This commit is contained in:
Deukhoofd
2020-10-04 17:15:28 +02:00
parent e0c52f4ae7
commit db7ad0bd76
6 changed files with 114 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
src/Parser/Lexer/Lexer.cpp
View File

@@ -215,6 +215,13 @@ namespace ElohimScript::Parser {
case u8'7':
case u8'8':
case u8'9': return LexNumerical(c);
case u8'\'': return LexString(u8'\'', false);
case u8'"': {
if (Peek() == '"' && Peek(2) == '\"') {
return LexString(u8'"', true);
}
return LexString(u8'"', false);
}
default: return new LexTokenImpl<LexTokenKind::Unknown>();
}
@@ -285,12 +292,12 @@ namespace ElohimScript::Parser {
while (true) {
auto v = (uint64_t)LexDecimalValue(Peek());
if (v == 255) {
if (!isDecimal && Peek() == '.') {
if (!isDecimal && Peek() == u8'.') {
isDecimal = true;
Progress();
continue;
}
if (isDecimal && (Peek() == 'e' || Peek() == 'E')) {
if (isDecimal && (Peek() == u8'e' || Peek() == u8'E')) {
isDecimal = false;
isExponent = true;
Progress();
@@ -316,12 +323,12 @@ namespace ElohimScript::Parser {
if (isExponent) {
val *= pow(10, exponentValue);
}
return new FloatToken(val);
return new FloatLiteral(val);
}
return new IntegerToken(value);
return new IntegerLiteral(value);
}
IntegerToken* Lexer::LexHexadecimal() {
IntegerLiteral* Lexer::LexHexadecimal() {
uint64_t value = 0;
while (true) {
auto v = LexHexadecimalValue(Peek());
@@ -332,9 +339,9 @@ namespace ElohimScript::Parser {
value <<= 4;
value += v;
}
return new IntegerToken(value);
return new IntegerLiteral(value);
}
IntegerToken* Lexer::LexOctal() {
IntegerLiteral* Lexer::LexOctal() {
uint64_t value = 0;
while (true) {
auto v = LexOctalValue(Peek());
@@ -345,9 +352,9 @@ namespace ElohimScript::Parser {
value <<= 3;
value += v;
}
return new IntegerToken(value);
return new IntegerLiteral(value);
}
IntegerToken* Lexer::LexBinary() {
IntegerLiteral* Lexer::LexBinary() {
uint64_t value = 0;
while (true) {
auto v = LexBinaryValue(Peek());
@@ -358,6 +365,38 @@ namespace ElohimScript::Parser {
value <<= 1;
value += v;
}
return new IntegerToken(value);
return new IntegerLiteral(value);
}
StringLiteral* Lexer::LexString(char8_t opening, bool heredoc) {
Progress();
if (heredoc) {
Progress(2);
}
auto start = _position;
size_t offset = 0;
while (true) {
auto current = Peek(offset);
if (heredoc) {
if (current == '"' && Peek(offset + 1) == '"' && Peek(offset + 2) == '"' && Peek(offset + 3) != '"') {
break;
}
} else if (current == opening) {
break;
}
if (current == u8'\0') {
// TODO: Log error
break;
}
if (!heredoc && (current == u8'\n' || current == u8'\r')) {
// TODO: log error
break;
}
offset++;
}
Progress(offset);
if (heredoc) {
Progress(2);
}
return new StringLiteral(std::u8string(_script.substr(start, offset)));
}
}