Deukhoofd
/
MalachScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes bug in identifier/keyword parsing, adds lexer integration tests.

This commit is contained in:
Deukhoofd 2020-10-04 21:05:51 +02:00
parent b6a5e047c2
commit 469c708788
Signed by: Deukhoofd
GPG Key ID: F63E044490819F6F
2 changed files with 31 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/Parser/Lexer/Lexer.cpp
View File

@ -466,9 +466,9 @@ namespace ElohimScript::Parser {
while (IsAlphaNumericalOrUnderscore(Peek(offset))) {
offset++;
}
auto str = _script.substr(_position, offset);
Progress(offset);
switch (Hash(str.data())) {
auto str = std::u8string(_script.substr(start, offset));
Progress(offset - 1);
switch (Hash(str.c_str())) {
case Hash(u8"and"): return new LexTokenImpl<LexTokenKind::AndKeyword>(TextSpan(start, _position));
case Hash(u8"abstract"): return new LexTokenImpl<LexTokenKind::AbstractKeyword>(TextSpan(start, _position));
case Hash(u8"auto"): return new LexTokenImpl<LexTokenKind::AutoKeyword>(TextSpan(start, _position));
@ -537,7 +537,7 @@ namespace ElohimScript::Parser {
case Hash(u8"while"): return new LexTokenImpl<LexTokenKind::WhileKeyword>(TextSpan(start, _position));
case Hash(u8"xor"): return new LexTokenImpl<LexTokenKind::XorKeyword>(TextSpan(start, _position));
default: return new IdentifierToken(TextSpan(start, _position), std::u8string(str));
default: return new IdentifierToken(TextSpan(start, _position), str);
}
}
bool Lexer::IsAlphaNumericalOrUnderscore(char8_t c) {

27
tests/LexerTests/LexerIntegrationTests.cpp Normal file
View File

@ -0,0 +1,27 @@
#include "../../extern/doctest.hpp"
#include "../../src/Parser/Lexer/Lexer.hpp"
using namespace ElohimScript::Parser;
#define LEX_TEST(script, ...) \
TEST_CASE("Lex: " script) { \
ElohimScript::Diagnostics::Diagnostics diag; \
auto lexer = Lexer(script, &diag); \
const auto* token = lexer.Lex(); \
CHECK(diag.GetMessages().empty()); \
std::vector<LexTokenKind> vec = {__VA_ARGS__, LexTokenKind::EndOfFile}; \
const auto* current = token; \
auto pos = 0; \
while (current != nullptr) { \
CHECK_MESSAGE(current->GetKind() == vec[pos], "position: " << pos); \
pos++; \
current = current->GetNext().get(); \
} \
delete token; \
}
LEX_TEST("1 + 1", LexTokenKind::IntegerLiteral, LexTokenKind::Whitespace, LexTokenKind::PlusSymbol,
LexTokenKind::Whitespace, LexTokenKind::IntegerLiteral);
LEX_TEST("private foo = \"foobar\"", LexTokenKind::PrivateKeyword, LexTokenKind::Whitespace, LexTokenKind::Identifier,
LexTokenKind::Whitespace, LexTokenKind::EqualsSymbol, LexTokenKind::Whitespace, LexTokenKind::StringLiteral);