#include #include #include "Lexer.hpp" Lexer::Lexer(string script) { Lexer::Script = std::move(script); Lexer::Position = 0; } vector Lexer::Lex() { vector tokens; while (true){ IToken* next = Lexer::LexNext(Lexer::Next()); tokens.push_back(next); if (next->GetKind() == TokenKind::EndOfFile) break; } return tokens; } char Lexer::Peek(){ if (Lexer::Position > Lexer::Script.length()) return '\0'; return Lexer::Script[Lexer::Position]; } char Lexer::Next(){ char next = Peek(); Lexer::Position++; return next; } IToken* Lexer::LexNext(char c){ switch (c) { case '\0': return new SimpleToken(TokenKind::EndOfFile); case ' ': case '\t': case '\n': case '\r': return new SimpleToken(TokenKind::WhiteSpace); case '+': return new SimpleToken(TokenKind::PlusToken); case '-': return new SimpleToken(TokenKind::MinusToken); case '/': return new SimpleToken(TokenKind::SlashToken); case '*': return new SimpleToken(TokenKind::StarToken); case '=': if (Lexer::Peek() == '='){ Lexer::Next(); return new SimpleToken(TokenKind::EqualityToken); } return new SimpleToken(TokenKind::AssignmentToken); case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return LexNumber(c); case '_': return LexIdentifierOrKeyword(c); default: if (isalpha(c)){ return LexIdentifierOrKeyword(c); } throw; } } int CharToInt(char c){ switch (c){ case '0': return 0; case '1': return 1; case '2': return 2; case '3': return 3; case '4': return 4; case '5': return 5; case '6': return 6; case '7': return 7; case '8': return 8; case '9': return 9; default: return -1; } } IToken* Lexer::LexNumber(char c){ long int_value = CharToInt(c); double float_value = 0; short decimal_index = 0; bool has_point = false; bool is_searching = true; while (is_searching){ char next = Lexer::Peek(); int next_val = CharToInt(next); if (next_val == -1){ switch (next){ case '_': Lexer::Next(); continue; case '.': Lexer::Next(); has_point = true; decimal_index = 0; float_value = int_value; continue; default: is_searching = false; continue; } } else{ Lexer::Next(); if (has_point){ decimal_index++; float_value += next_val / pow(10, decimal_index); } else { int_value *= 10; int_value += next_val; } } } if (has_point){ return new FloatToken(float_value); } else{ return new IntegerToken(int_value); } } unsigned constexpr const_hash(char const *input) { return *input ? static_cast(*input) + 33 * const_hash(input + 1) : 5381; } IToken* Lexer::LexIdentifierOrKeyword(char c){ vector charVec(1, c); while (true){ char next = Lexer::Peek(); if (next == '\0') break; if (isalpha(next) || next == '_'){ Lexer::Next(); charVec.push_back(next); } else{ break; } } string s = string(charVec.begin(), charVec.end()); switch (const_hash(s.c_str())){ case const_hash("and"): return new SimpleToken(TokenKind::AndKeyword); case const_hash("break"): return new SimpleToken(TokenKind::BreakKeyword); case const_hash("do"): return new SimpleToken(TokenKind::DoKeyword); case const_hash("else"): return new SimpleToken(TokenKind::ElseKeyword); case const_hash("elseif"): return new SimpleToken(TokenKind::ElseIfKeyword); case const_hash("end"): return new SimpleToken(TokenKind::EndKeyword); case const_hash("false"): return new SimpleToken(TokenKind::FalseKeyword); case const_hash("for"): return new SimpleToken(TokenKind::ForKeyword); case const_hash("function"): return new SimpleToken(TokenKind::FunctionKeyword); case const_hash("if"): return new SimpleToken(TokenKind::IfKeyword); case const_hash("in"): return new SimpleToken(TokenKind::InKeyword); case const_hash("local"): return new SimpleToken(TokenKind::LocalKeyword); case const_hash("nil"): return new SimpleToken(TokenKind::NilKeyword); case const_hash("not"): return new SimpleToken(TokenKind::NotKeyword); case const_hash("or"): return new SimpleToken(TokenKind::OrKeyword); case const_hash("return"): return new SimpleToken(TokenKind::ReturnKeyword); case const_hash("then"): return new SimpleToken(TokenKind::ThenKeyword); case const_hash("true"): return new SimpleToken(TokenKind::TrueKeyword); case const_hash("while"): return new SimpleToken(TokenKind::WhileKeyword); default: return new IdentifierToken(s); } }