Implements string lexing.

This commit is contained in:
2020-10-04 17:15:28 +02:00
parent e0c52f4ae7
commit db7ad0bd76
6 changed files with 114 additions and 27 deletions

View File

@@ -8,6 +8,7 @@ namespace ElohimScript::Parser {
class Lexer {
public:
Lexer(const char* script) : _script(reinterpret_cast<const char8_t*>(script)) {}
Lexer(const char8_t* script) : _script(script) {}
Lexer(std::u8string_view script) : _script(script) {}
const LexToken* Lex();
@@ -37,9 +38,11 @@ namespace ElohimScript::Parser {
LexToken* LexNext();
LexToken* LexNumerical(char8_t);
LexToken* LexDecimal(uint64_t initial);
IntegerToken* LexHexadecimal();
IntegerToken* LexOctal();
IntegerToken* LexBinary();
IntegerLiteral* LexHexadecimal();
IntegerLiteral* LexOctal();
IntegerLiteral* LexBinary();
StringLiteral* LexString(char8_t opening, bool heredoc);
};
}