Make Lexer use constant Tokens

src/Parser/Lexer.cpp

@@ -14,8 +14,8 @@ Lexer::Lexer(const string& scriptString, class Script* script)
 }
 
 
-vector<IToken*> Lexer::Lex() {
-    vector<IToken*> tokens;
+vector<const IToken*> Lexer::Lex() {
+    vector<const IToken*> tokens;
     while (true){
         IToken* next = this -> LexNext(this -> Next());
         auto nextKind = next -> GetKind();
@@ -215,11 +215,11 @@ IToken * Lexer::LexIdentifierOrKeyword() {
         case HashedString::ConstHash("then"): return new SimpleToken(TokenKind::ThenKeyword, start, 4);
         case HashedString::ConstHash("true"): return new SimpleToken(TokenKind::TrueKeyword, start, 4);
         case HashedString::ConstHash("while"): return new SimpleToken(TokenKind::WhileKeyword, start, 5);
-        default: return new IdentifierToken(s, start, s.length());
+        default: return new IdentifierToken(HashedString(s), start, s.length());
     }
 }
 
-const unordered_map<char, char> ControlCharacters{
+const unordered_map<char, char> ControlCharacters{ // NOLINT(cert-err58-cpp)
         {'0', '\0'},
         {'a', '\a'},
         {'b', '\b'},

src/Parser/Lexer.hpp

@@ -23,7 +23,7 @@ public:
 public:
     Script* ScriptData;
 
-    vector<IToken*> Lex();
+    vector<const IToken*> Lex();
     explicit Lexer(const string& scriptString, class Script* script);
 
 };

src/Parser/ParsedExpressions/ParsedExpression.hpp

@@ -73,7 +73,7 @@ public:
     }
     explicit LiteralIntegerExpression(IntegerToken* token)
         : ParsedExpression(token -> GetStartPosition(), token -> GetLength()),
-          _value(token -> Value)
+          _value(token -> GetValue())
     {
     }
 
@@ -90,7 +90,7 @@ public:
     }
     explicit LiteralFloatExpression(FloatToken* token)
         : ParsedExpression(token -> GetStartPosition(), token -> GetLength()),
-          _value(token -> Value)
+          _value(token -> GetValue())
     {
     }
 
@@ -107,7 +107,7 @@ public:
     }
     explicit LiteralStringExpression(StringToken* token)
         : ParsedExpression(token -> GetStartPosition(), token -> GetLength()),
-          _value(std::move(token -> Value))
+          _value(std::move(token -> GetValue()))
     {
     }
 
@@ -122,7 +122,7 @@ public:
     const ParsedExpressionKind GetKind() const final{
         return ParsedExpressionKind::LiteralBool;
     }
-    explicit LiteralBoolExpression(IToken* token)
+    explicit LiteralBoolExpression(const IToken* token)
         : ParsedExpression(token -> GetStartPosition(), token -> GetLength()),
           _value(token -> GetKind() == TokenKind::TrueKeyword)
     {
@@ -140,7 +140,7 @@ public:
         return ParsedExpressionKind::Variable;
     }
     explicit VariableExpression(IdentifierToken* token) : ParsedExpression(token -> GetStartPosition(), token -> GetLength())
-        , _value(HashedString(token -> Value))
+        , _value(HashedString(token -> GetValue()))
     {
     }
 

src/Parser/ParsedStatements/ParsedStatement.hpp

@@ -154,8 +154,8 @@ class ParsedAssignmentStatement : public ParsedStatement{
     const HashedString _identifier;
     const ParsedExpression* _expression;
 public:
-    ParsedAssignmentStatement(bool local, const std::string& identifier, ParsedExpression* expression, unsigned int start, unsigned int length)
-    : ParsedStatement(start, length), _local(local), _identifier(HashedString(identifier)), _expression(expression)
+    ParsedAssignmentStatement(bool local, const HashedString identifier, ParsedExpression* expression, unsigned int start, unsigned int length)
+    : ParsedStatement(start, length), _local(local), _identifier(identifier), _expression(expression)
     {
     }
 

src/Parser/Parser.cpp

@@ -22,20 +22,20 @@ ParsedScriptStatement* Parser::Parse() {
     return new ParsedScriptStatement(statements);
 }
 
-IToken *Parser::Peek() {
+const IToken *Parser::Peek() {
     return this -> _tokens[_position];
 }
 
-IToken *Parser::PeekAt(int offset) {
+const IToken *Parser::PeekAt(int offset) {
     return this -> _tokens[_position + offset];
 }
 
-IToken *Parser::Next() {
+const IToken *Parser::Next() {
     this -> _position++;
     return this -> _tokens[_position - 1];
 }
 
-ParsedStatement* Parser::ParseStatement(IToken* current){
+ParsedStatement* Parser::ParseStatement(const IToken* current){
     auto currentKind = current->GetKind();
     switch (currentKind){
         case TokenKind ::LocalKeyword: return this -> ParseAssignment(current);
@@ -50,9 +50,9 @@ ParsedStatement* Parser::ParseStatement(IToken* current){
     return new ParsedExpressionStatement(this -> ParseExpression(current));
 }
 
-ParsedStatement *Parser::ParseAssignment(IToken *current) {
+ParsedStatement *Parser::ParseAssignment(const IToken *current) {
     bool isLocal = false;
-    IToken* identifier;
+    const IToken* identifier;
     if (current -> GetKind() == TokenKind::LocalKeyword){
         isLocal = true;
         identifier = this -> Next();
@@ -72,7 +72,7 @@ ParsedStatement *Parser::ParseAssignment(IToken *current) {
     }
 
     auto start = current -> GetStartPosition();
-    return new ParsedAssignmentStatement(isLocal, ((IdentifierToken*)identifier) -> Value, expression, start, expression->GetEndPosition() - start);
+    return new ParsedAssignmentStatement(isLocal, ((IdentifierToken*)identifier) -> GetValue(), expression, start, expression->GetEndPosition() - start);
 }
 
 ParsedStatement *Parser::ParseBlock(const vector<TokenKind>& endTokens, const vector<const ParsedStatement*>& openStatements) {
@@ -96,7 +96,7 @@ ParsedStatement *Parser::ParseBlock(const vector<TokenKind>& endTokens, const ve
     return new ParsedBlockStatement(statements);
 }
 
-ParsedStatement *Parser::ParseFunctionDeclaration(IToken *current) {
+ParsedStatement *Parser::ParseFunctionDeclaration(const IToken *current) {
     auto functionIdentifierToken = this->Next();
     auto openParenthesis = this->Next();
     vector<TypedVariableIdentifier*> parameters;
@@ -135,7 +135,7 @@ ParsedStatement *Parser::ParseFunctionDeclaration(IToken *current) {
         }
         auto typeToken = (IdentifierToken*)type;
         auto identifierToken = (IdentifierToken*)identifier;
-        parameters.push_back(new TypedVariableIdentifier(HashedString(typeToken->Value), HashedString(identifierToken->Value)));
+        parameters.push_back(new TypedVariableIdentifier(typeToken->GetValue(), typeToken->GetValue()));
 
         auto nextKind = next->GetKind();
         if (nextKind == TokenKind::CloseParenthesis || nextKind == TokenKind::EndOfFile){
@@ -154,18 +154,18 @@ ParsedStatement *Parser::ParseFunctionDeclaration(IToken *current) {
     if (block->GetKind() == ParsedStatementKind::Bad){
         return new ParsedBadStatement(start, block->GetEndPosition() - start);
     }
-    auto functionIdentifier = ((IdentifierToken*) functionIdentifierToken)->Value;
+    auto functionIdentifier = ((IdentifierToken*) functionIdentifierToken)->GetValue();
     return new ParsedFunctionDeclarationStatement(HashedString(functionIdentifier), parameters, (ParsedBlockStatement*)block, start, block->GetEndPosition() - start);
 }
 
-ParsedStatement* Parser::ParseReturnStatement(IToken* current){
+ParsedStatement* Parser::ParseReturnStatement(const IToken* current){
     //TODO: if next token is on a different line, don't parse it as return expression.
     auto expression = this->ParseExpression(this->Next());
     auto start = current->GetStartPosition();
     return new ParsedReturnStatement(expression, start, expression->GetEndPosition() - start);
 }
 
-ParsedStatement* Parser::ParseIfStatement(IToken* current){
+ParsedStatement* Parser::ParseIfStatement(const IToken* current){
     auto condition = this->ParseExpression(this->Next());
     auto next = this -> Next();
     if (next->GetKind() != TokenKind::ThenKeyword){
@@ -185,7 +185,7 @@ ParsedStatement* Parser::ParseIfStatement(IToken* current){
     return new ParsedConditionalStatement(condition, block, start, block->GetEndPosition() - start);
 }
 
-ParsedExpression* Parser::ParseExpression(IToken* current){
+ParsedExpression* Parser::ParseExpression(const IToken* current){
     auto expression = this -> ParseBinaryExpression(current, OperatorPrecedence::No);
     auto peekKind = this->Peek()->GetKind();
     while (peekKind == TokenKind::OpenParenthesis ||
@@ -274,7 +274,7 @@ OperatorPrecedence GetBinaryPrecedence(TokenKind kind){
     }
 }
 
-ParsedExpression* Parser::ParseBinaryExpression(IToken* current, OperatorPrecedence parentPrecedence){
+ParsedExpression* Parser::ParseBinaryExpression(const IToken* current, OperatorPrecedence parentPrecedence){
     OperatorPrecedence unaryPrecedence = GetUnaryPrecedence(current -> GetKind());
     ParsedExpression* left;
     if (unaryPrecedence != OperatorPrecedence::No && unaryPrecedence >= parentPrecedence){
@@ -301,7 +301,7 @@ ParsedExpression* Parser::ParseBinaryExpression(IToken* current, OperatorPrecede
     return left;
 }
 
-ParsedExpression *Parser::ParsePrimaryExpression(IToken *current) {
+ParsedExpression *Parser::ParsePrimaryExpression(const IToken *current) {
     switch (current -> GetKind()){
         case TokenKind ::Integer: return new LiteralIntegerExpression((IntegerToken*)current);
         case TokenKind ::Float: return new LiteralFloatExpression((FloatToken*)current);
@@ -319,7 +319,7 @@ ParsedExpression *Parser::ParsePrimaryExpression(IToken *current) {
     }
 }
 
-ParsedExpression *Parser::ParseParenthesizedExpression(IToken *current) {
+ParsedExpression *Parser::ParseParenthesizedExpression(const IToken *current) {
     auto next = this -> Next();
     auto expression = this -> ParseExpression(next);
     auto closeToken = this -> Next();
@@ -369,7 +369,7 @@ ParsedExpression* Parser::ParseIndexExpression(ParsedExpression* indexingExpress
     return new IndexExpression(indexingExpression, indexExpression, start, closeBracket->GetEndPosition() - start);
 }
 
-ParsedExpression* Parser::ParseTableExpression(IToken* current){
+ParsedExpression* Parser::ParseTableExpression(const IToken* current){
     if (this -> Peek() -> GetKind() == TokenKind::CloseCurlyBracket){
         this -> Next();
         auto start = current->GetStartPosition();

src/Parser/Parser.hpp

@@ -18,39 +18,36 @@ enum class OperatorPrecedence {
 };
 
 class Parser {
-    vector<IToken*> _tokens;
+    vector<const IToken*> _tokens;
     unsigned int _position;
     Script* ScriptData;
 
-    IToken* Peek();
-    IToken* Next();
+    const IToken* Peek();
+    const IToken *PeekAt(int offset);
+    const IToken* Next();
 
-    ParsedStatement* ParseStatement(IToken* current);
-    ParsedStatement* ParseAssignment(IToken* current);
+    ParsedStatement* ParseStatement(const IToken* current);
+    ParsedStatement* ParseAssignment(const IToken* current);
     ParsedStatement *ParseBlock(const vector<TokenKind>& endTokens, const vector<const ParsedStatement*>& openStatements = {});
-    ParsedStatement* ParseFunctionDeclaration(IToken* current);
-    ParsedStatement *ParseReturnStatement(IToken *current);
-    ParsedStatement *ParseIfStatement(IToken *current);
+    ParsedStatement* ParseFunctionDeclaration(const IToken* current);
+    ParsedStatement *ParseReturnStatement(const IToken *current);
+    ParsedStatement *ParseIfStatement(const IToken *current);
 
-    ParsedExpression* ParseExpression(IToken* current);
-    ParsedExpression* ParseBinaryExpression(IToken* current, OperatorPrecedence parentPrecedence);
-    ParsedExpression* ParsePrimaryExpression(IToken* current);
-    ParsedExpression* ParseParenthesizedExpression(IToken *current);
+    ParsedExpression* ParseExpression(const IToken* current);
+    ParsedExpression* ParseBinaryExpression(const IToken* current, OperatorPrecedence parentPrecedence);
+    ParsedExpression* ParsePrimaryExpression(const IToken* current);
+    ParsedExpression* ParseParenthesizedExpression(const IToken *current);
 
     ParsedExpression* ParseFunctionCallExpression(ParsedExpression* functionExpression);
     ParsedExpression *ParseIndexExpression(ParsedExpression *indexingExpression);
+    ParsedExpression *ParseTableExpression(const IToken *current);
 public:
     ParsedScriptStatement* Parse();
-    explicit Parser(const vector<IToken*>& tokens, Script* scriptData){
+    explicit Parser(const vector<const IToken*>& tokens, Script* scriptData){
         _tokens = tokens;
         _position = 0;
         ScriptData = scriptData;
     }
-
-
-    IToken *PeekAt(int offset);
-
-    ParsedExpression *ParseTableExpression(IToken *current);
 };
 
 

src/Parser/Token.hpp

@@ -1,104 +1,131 @@
 #include <utility>
 
+
 #ifndef PORYGONLANG_TOKEN_HPP
 #define PORYGONLANG_TOKEN_HPP
 
 #include <utility>
 #include <string>
 #include "TokenKind.hpp"
+#include "../Utilities/HashedString.hpp"
+
 using namespace std;
 
 class IToken{
-    unsigned int Position;
-    unsigned int Length;
+    const unsigned int _position;
+    const unsigned int _length;
 public:
-    virtual TokenKind GetKind() = 0;
+    virtual const TokenKind GetKind() const = 0;
 
-    IToken(unsigned int position, unsigned int length){
-        Position = position;
-        Length = length;
+    IToken(unsigned int position, unsigned int length)
+        : _position(position), _length(length)
+    {
     }
 
-    unsigned int GetStartPosition(){
-        return Position;
+    const unsigned int GetStartPosition() const{
+        return _position;
     }
 
-    unsigned int GetEndPosition(){
-        return Position + Length - 1;
+    const unsigned int GetEndPosition() const{
+        return _position + _length - 1;
     }
 
-    unsigned int GetLength(){
-        return Length;
+    const unsigned int GetLength() const{
+        return _length;
     }
 
-    virtual ~IToken(){
-
-    }
+    virtual ~IToken() = default;
 };
 
 class SimpleToken : public IToken{
+    const TokenKind _kind;
 public:
-    TokenKind Kind;
 
-    explicit SimpleToken(TokenKind type, unsigned int position, unsigned int length) : IToken(position, length){
-        Kind = type;
+    explicit SimpleToken(TokenKind kind, unsigned int position, unsigned int length)
+        : IToken(position, length),
+          _kind(kind)
+    {
     }
 
-    TokenKind GetKind() override{
-        return Kind;
+    const TokenKind GetKind() const final{
+        return _kind;
     }
 };
 
 class IntegerToken : public IToken{
+    const long _value;
 public:
-    long Value;
 
-    explicit IntegerToken(long value, unsigned int position, unsigned int length) : IToken(position, length){
-        Value = value;
+    explicit IntegerToken(long value, unsigned int position, unsigned int length)
+        : IToken(position, length),
+          _value(value)
+    {
     }
 
-    TokenKind GetKind() override{
+    const TokenKind GetKind() const final{
         return TokenKind::Integer;
     }
+
+    const long GetValue() const{
+        return _value;
+    }
 };
 
 class FloatToken : public IToken{
+    const double _value;
 public:
-    double Value;
 
-    explicit FloatToken(double value, unsigned int position, unsigned int length) : IToken(position, length){
-        Value = value;
+    explicit FloatToken(double value, unsigned int position, unsigned int length)
+        : IToken(position, length),
+          _value(value)
+    {
     }
 
-    TokenKind GetKind() override{
+    const TokenKind GetKind() const final{
         return TokenKind::Float;
     }
+
+    const double GetValue() const{
+        return _value;
+    }
 };
 
 class StringToken : public IToken{
+    const string _value;
 public:
-    string Value;
 
-    explicit StringToken(string value, unsigned int position, unsigned int length) : IToken(position, length){
-        Value = std::move(value);
+    explicit StringToken(string value, unsigned int position, unsigned int length)
+        : IToken(position, length),
+          _value(std::move(value))
+    {
     }
 
-    TokenKind GetKind() override{
+    const TokenKind GetKind() const final{
         return TokenKind::String;
     }
+
+    const string& GetValue() const{
+        return _value;
+    }
 };
 
 class IdentifierToken : public IToken{
+    const HashedString _value;
 public:
-    string Value;
 
-    explicit IdentifierToken(string value, unsigned int position, unsigned int length) : IToken(position, length){
-        Value = std::move(value);
+    explicit IdentifierToken(const HashedString value, unsigned int position, unsigned int length)
+        : IToken(position, length),
+          _value(value)
+    {
     }
 
-    TokenKind GetKind() override{
+    const TokenKind GetKind() const final{
         return TokenKind::Identifier;
     }
+
+    const HashedString GetValue() const{
+        return _value;
+    }
 };
 
 #endif //PORYGONLANG_TOKEN_HPP

src/Utilities/HashedString.hpp

@@ -11,6 +11,7 @@ public:
     }
     explicit HashedString(char const *input) : _hash(ConstHash(input)){
     }
+    HashedString(const HashedString& b) = default;
 
     static unsigned constexpr ConstHash(char const *input) {
         return *input ?