Fixes bug in identifier/keyword parsing, adds lexer integration tests.
This commit is contained in:
parent
b6a5e047c2
commit
469c708788
|
@ -466,9 +466,9 @@ namespace ElohimScript::Parser {
|
|||
while (IsAlphaNumericalOrUnderscore(Peek(offset))) {
|
||||
offset++;
|
||||
}
|
||||
auto str = _script.substr(_position, offset);
|
||||
Progress(offset);
|
||||
switch (Hash(str.data())) {
|
||||
auto str = std::u8string(_script.substr(start, offset));
|
||||
Progress(offset - 1);
|
||||
switch (Hash(str.c_str())) {
|
||||
case Hash(u8"and"): return new LexTokenImpl<LexTokenKind::AndKeyword>(TextSpan(start, _position));
|
||||
case Hash(u8"abstract"): return new LexTokenImpl<LexTokenKind::AbstractKeyword>(TextSpan(start, _position));
|
||||
case Hash(u8"auto"): return new LexTokenImpl<LexTokenKind::AutoKeyword>(TextSpan(start, _position));
|
||||
|
@ -537,7 +537,7 @@ namespace ElohimScript::Parser {
|
|||
case Hash(u8"while"): return new LexTokenImpl<LexTokenKind::WhileKeyword>(TextSpan(start, _position));
|
||||
case Hash(u8"xor"): return new LexTokenImpl<LexTokenKind::XorKeyword>(TextSpan(start, _position));
|
||||
|
||||
default: return new IdentifierToken(TextSpan(start, _position), std::u8string(str));
|
||||
default: return new IdentifierToken(TextSpan(start, _position), str);
|
||||
}
|
||||
}
|
||||
bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) {
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
#include "../../extern/doctest.hpp"
|
||||
#include "../../src/Parser/Lexer/Lexer.hpp"
|
||||
|
||||
using namespace ElohimScript::Parser;
|
||||
|
||||
#define LEX_TEST(script, ...) \
|
||||
TEST_CASE("Lex: " script) { \
|
||||
ElohimScript::Diagnostics::Diagnostics diag; \
|
||||
auto lexer = Lexer(script, &diag); \
|
||||
const auto* token = lexer.Lex(); \
|
||||
CHECK(diag.GetMessages().empty()); \
|
||||
std::vector<LexTokenKind> vec = {__VA_ARGS__, LexTokenKind::EndOfFile}; \
|
||||
const auto* current = token; \
|
||||
auto pos = 0; \
|
||||
while (current != nullptr) { \
|
||||
CHECK_MESSAGE(current->GetKind() == vec[pos], "position: " << pos); \
|
||||
pos++; \
|
||||
current = current->GetNext().get(); \
|
||||
} \
|
||||
delete token; \
|
||||
}
|
||||
|
||||
LEX_TEST("1 + 1", LexTokenKind::IntegerLiteral, LexTokenKind::Whitespace, LexTokenKind::PlusSymbol,
|
||||
LexTokenKind::Whitespace, LexTokenKind::IntegerLiteral);
|
||||
|
||||
LEX_TEST("private foo = \"foobar\"", LexTokenKind::PrivateKeyword, LexTokenKind::Whitespace, LexTokenKind::Identifier,
|
||||
LexTokenKind::Whitespace, LexTokenKind::EqualsSymbol, LexTokenKind::Whitespace, LexTokenKind::StringLiteral);
|
Loading…
Reference in New Issue