Implements string lexing.

src/Parser/Lexer/LexToken.hpp

@@ -2,6 +2,7 @@
 #define ELOHIMSCRIPT_LEXTOKEN_HPP
 
 #include <memory>
+#include <utility>
 #include "LexTokenKind.hpp"
 
 namespace ElohimScript::Parser {
@@ -22,21 +23,29 @@ namespace ElohimScript::Parser {
         [[nodiscard]] LexTokenKind GetKind() const noexcept override { return kind; }
     };
 
-    class IntegerToken : public LexTokenImpl<LexTokenKind::IntegerToken> {
+    class IntegerLiteral : public LexTokenImpl<LexTokenKind::IntegerLiteral> {
         uint64_t _value;
 
     public:
-        IntegerToken(uint64_t value) : _value(value) {}
+        IntegerLiteral(uint64_t value) : _value(value) {}
         [[nodiscard]] uint64_t GetValue() const noexcept { return _value; }
     };
 
-    class FloatToken : public LexTokenImpl<LexTokenKind::FloatToken> {
+    class FloatLiteral : public LexTokenImpl<LexTokenKind::FloatLiteral> {
         double _value;
 
     public:
-        FloatToken(double value) : _value(value) {}
+        FloatLiteral(double value) : _value(value) {}
         [[nodiscard]] double GetValue() const noexcept { return _value; }
     };
+
+    class StringLiteral : public LexTokenImpl<LexTokenKind::StringLiteral> {
+        std::u8string _value;
+
+    public:
+        StringLiteral(std::u8string value) : _value(std::move(value)) {}
+        [[nodiscard]] const std::u8string& GetValue() const noexcept { return _value; }
+    };
 }
 
 #endif // ELOHIMSCRIPT_LEXTOKEN_HPP

src/Parser/Lexer/LexTokenKind.hpp

@@ -62,9 +62,10 @@ namespace ElohimScript::Parser {
         ExclamationMarkIsSymbol,
         ColonColonSymbol,
 
-        // Misc
-        FloatToken,
-        IntegerToken,
+        // Literals
+        FloatLiteral,
+        IntegerLiteral,
+        StringLiteral,
     };
 }
 

src/Parser/Lexer/Lexer.cpp

@@ -215,6 +215,13 @@ namespace ElohimScript::Parser {
             case u8'7':
             case u8'8':
             case u8'9': return LexNumerical(c);
+            case u8'\'': return LexString(u8'\'', false);
+            case u8'"': {
+                if (Peek() == '"' && Peek(2) == '\"') {
+                    return LexString(u8'"', true);
+                }
+                return LexString(u8'"', false);
+            }
 
             default: return new LexTokenImpl<LexTokenKind::Unknown>();
         }
@@ -285,12 +292,12 @@ namespace ElohimScript::Parser {
         while (true) {
             auto v = (uint64_t)LexDecimalValue(Peek());
             if (v == 255) {
-                if (!isDecimal && Peek() == '.') {
+                if (!isDecimal && Peek() == u8'.') {
                     isDecimal = true;
                     Progress();
                     continue;
                 }
-                if (isDecimal && (Peek() == 'e' || Peek() == 'E')) {
+                if (isDecimal && (Peek() == u8'e' || Peek() == u8'E')) {
                     isDecimal = false;
                     isExponent = true;
                     Progress();
@@ -316,12 +323,12 @@ namespace ElohimScript::Parser {
             if (isExponent) {
                 val *= pow(10, exponentValue);
             }
-            return new FloatToken(val);
+            return new FloatLiteral(val);
         }
-        return new IntegerToken(value);
+        return new IntegerLiteral(value);
     }
 
-    IntegerToken* Lexer::LexHexadecimal() {
+    IntegerLiteral* Lexer::LexHexadecimal() {
         uint64_t value = 0;
         while (true) {
             auto v = LexHexadecimalValue(Peek());
@@ -332,9 +339,9 @@ namespace ElohimScript::Parser {
             value <<= 4;
             value += v;
         }
-        return new IntegerToken(value);
+        return new IntegerLiteral(value);
     }
-    IntegerToken* Lexer::LexOctal() {
+    IntegerLiteral* Lexer::LexOctal() {
         uint64_t value = 0;
         while (true) {
             auto v = LexOctalValue(Peek());
@@ -345,9 +352,9 @@ namespace ElohimScript::Parser {
             value <<= 3;
             value += v;
         }
-        return new IntegerToken(value);
+        return new IntegerLiteral(value);
     }
-    IntegerToken* Lexer::LexBinary() {
+    IntegerLiteral* Lexer::LexBinary() {
         uint64_t value = 0;
         while (true) {
             auto v = LexBinaryValue(Peek());
@@ -358,6 +365,38 @@ namespace ElohimScript::Parser {
             value <<= 1;
             value += v;
         }
-        return new IntegerToken(value);
+        return new IntegerLiteral(value);
+    }
+    StringLiteral* Lexer::LexString(char8_t opening, bool heredoc) {
+        Progress();
+        if (heredoc) {
+            Progress(2);
+        }
+        auto start = _position;
+        size_t offset = 0;
+        while (true) {
+            auto current = Peek(offset);
+            if (heredoc) {
+                if (current == '"' && Peek(offset + 1) == '"' && Peek(offset + 2) == '"' && Peek(offset + 3) != '"') {
+                    break;
+                }
+            } else if (current == opening) {
+                break;
+            }
+            if (current == u8'\0') {
+                // TODO: Log error
+                break;
+            }
+            if (!heredoc && (current == u8'\n' || current == u8'\r')) {
+                // TODO: log error
+                break;
+            }
+            offset++;
+        }
+        Progress(offset);
+        if (heredoc) {
+            Progress(2);
+        }
+        return new StringLiteral(std::u8string(_script.substr(start, offset)));
     }
 }

src/Parser/Lexer/Lexer.hpp

@@ -8,6 +8,7 @@ namespace ElohimScript::Parser {
     class Lexer {
     public:
         Lexer(const char* script) : _script(reinterpret_cast<const char8_t*>(script)) {}
+        Lexer(const char8_t* script) : _script(script) {}
         Lexer(std::u8string_view script) : _script(script) {}
         const LexToken* Lex();
 
@@ -37,9 +38,11 @@ namespace ElohimScript::Parser {
         LexToken* LexNext();
         LexToken* LexNumerical(char8_t);
         LexToken* LexDecimal(uint64_t initial);
-        IntegerToken* LexHexadecimal();
-        IntegerToken* LexOctal();
-        IntegerToken* LexBinary();
+        IntegerLiteral* LexHexadecimal();
+        IntegerLiteral* LexOctal();
+        IntegerLiteral* LexBinary();
+
+        StringLiteral* LexString(char8_t opening, bool heredoc);
     };
 }