It is finally time to commit the entire syntax highlighting code and first is always the pattern language

lib/include/pl/core/lexer.hpp

@@ -26,7 +26,7 @@ namespace pl::core {
         bool processToken(auto parserFunction, const std::string_view& identifier);
         Location location() override;
 
-        std::optional<char> parseCharacter();
+        std::optional<char> parseCharacter(bool isWindowsPath=false);
         std::optional<Token> parseOperator();
         std::optional<Token> parseSeparator();
         std::optional<Token> parseOneLineComment();
@@ -40,7 +40,7 @@ namespace pl::core {
         std::optional<Token> parseConstant(const std::string_view &identifier);
         std::optional<Token> parseStringLiteral();
         std::optional<Token> parseDirectiveArgument();
-        std::optional<Token> parseDirectiveValue();
+        std::optional<Token> parseDirectiveValue(bool forInclude=false);
         std::optional<Token::Literal> parseIntegerLiteral(std::string_view literal);
 
         std::optional<double> parseFloatingPoint(std::string_view literal, char suffix);

lib/include/pl/core/parser.hpp

@@ -149,10 +149,10 @@ namespace pl::core {
         hlp::safe_unique_ptr<ast::ASTNode> parseUnaryExpression();
         hlp::safe_unique_ptr<ast::ASTNode> parseMultiplicativeExpression();
         hlp::safe_unique_ptr<ast::ASTNode> parseAdditiveExpression();
-        hlp::safe_unique_ptr<ast::ASTNode> parseShiftExpression();
-        hlp::safe_unique_ptr<ast::ASTNode> parseBinaryAndExpression();
-        hlp::safe_unique_ptr<ast::ASTNode> parseBinaryXorExpression();
-        hlp::safe_unique_ptr<ast::ASTNode> parseBinaryOrExpression(bool inMatchRange);
+        hlp::safe_unique_ptr<ast::ASTNode> parseShiftExpression(bool inTemplate);
+        hlp::safe_unique_ptr<ast::ASTNode> parseBinaryAndExpression(bool inTemplate);
+        hlp::safe_unique_ptr<ast::ASTNode> parseBinaryXorExpression(bool inTemplate);
+        hlp::safe_unique_ptr<ast::ASTNode> parseBinaryOrExpression(bool inTemplate, bool inMatchRange);
         hlp::safe_unique_ptr<ast::ASTNode> parseBooleanAnd(bool inTemplate, bool inMatchRange);
         hlp::safe_unique_ptr<ast::ASTNode> parseBooleanXor(bool inTemplate, bool inMatchRange);
         hlp::safe_unique_ptr<ast::ASTNode> parseBooleanOr(bool inTemplate, bool inMatchRange);

lib/include/pl/core/preprocessor.hpp

@@ -30,35 +30,59 @@ namespace pl::core {
         hlp::CompileResult<std::vector<Token>> preprocess(PatternLanguage *runtime, api::Source* source, bool initialRun = true);
 
         void addDefine(const std::string &name, const std::string &value = "");
+        void removeDefine(const std::string &name);
         void addPragmaHandler(const std::string &pragmaType, const api::PragmaHandler &handler);
         void addDirectiveHandler(const Token::Directive &directiveType, const api::DirectiveHandler &handler);
         void removePragmaHandler(const std::string &pragmaType);
         void removeDirectiveHandler(const Token::Directive &directiveType);
-        void validateExcludedLocations();
-        void appendExcludedLocation(const ExcludedLocation &location);
+
         void validateOutput();
 
-        [[nodiscard]] auto getExcludedLocations() const {
+        [[nodiscard]] const std::vector<ExcludedLocation> &getExcludedLocations() const {
             return m_excludedLocations;
         }
 
-        [[nodiscard]] auto getResult() const {
-            return this->m_result;
+        [[nodiscard]] const std::vector<Token> &getResult() {
+            return m_result;
         }
 
         [[nodiscard]] auto getOutput() const {
             return this->m_output;
         }
 
-        void setOutput(std::vector<pl::core::Token> tokens) {
-            m_output = tokens;
+        void setOutput(const std::vector<pl::core::Token> &tokens) {
+            u32 j =0;
+            auto tokenCount = m_result.size();
+            for (auto token : tokens) {
+                if (auto identifier = std::get_if<Token::Identifier>(&token.value); identifier != nullptr) {
+                    if (auto type = identifier->getType(); type > Token::Identifier::IdentifierType::ScopeResolutionUnknown) {
+                        auto location = token.location;
+                        if (location.source->source != "<Source Code>")
+                            continue;
+                        auto line = location.line;
+                        auto column = location.column;
+                        while (m_result[j].location.line < line) {
+                            if (j >= tokenCount)
+                                break;
+                            j++;
+                        }
+                        while (m_result[j].location.column < column) {
+                            if (j >= tokenCount)
+                                break;
+                            j++;
+                        }
+                        if (auto identifier2 = std::get_if<Token::Identifier>(&m_result[j].value); identifier2 != nullptr)
+                            identifier2->setType(type);
+                    }
+                }
+            }
         }
 
-        [[nodiscard]] auto getErrors() const {
+        [[nodiscard]] const std::vector<err::CompileError> &getErrors() const {
             return this->m_errors;
         }
 
-        void setErrors(std::vector<err::CompileError> errors) {
+        void setErrors(const std::vector<err::CompileError> &errors) {
             m_errors = errors;
         }
 
@@ -70,11 +94,15 @@ namespace pl::core {
             return m_initialized;
         }
 
+        [[nodiscard]] const api::Resolver& getResolver() const {
+            return m_resolver;
+        }
+
         void setResolver(const api::Resolver& resolvers) {
             m_resolver = resolvers;
         }
 
-        auto getNamespaces() const {
+        const std::vector<std::string> getNamespaces() const {
             return m_namespaces;
         }
 
@@ -85,6 +113,7 @@ namespace pl::core {
         bool eof();
         Location location() override;
         void removeKey(const Token &token);
+        void nextLine(u32 line);
         // directive handlers
         void handleIfDef(u32 line);
         void handleIfNDef(u32 line);
@@ -103,6 +132,7 @@ namespace pl::core {
         std::unordered_map<Token::Directive, api::DirectiveHandler> m_directiveHandlers;
 
         std::unordered_map<std::string, std::vector<Token>> m_defines;
+        std::map<std::string, std::string> m_addedDefines;
         std::unordered_map<std::string, std::vector<std::pair<std::string, u32>>> m_pragmas;
         std::vector<ExcludedLocation> m_excludedLocations;
 

lib/include/pl/core/token.hpp

@@ -179,10 +179,11 @@ namespace pl::core {
                 Typedef,
                 Function,
                 UDT,
+                View,
                 FunctionVariable,
                 FunctionParameter,
-                PatternLocalVariable,
-                PatternPlacedVariable,
+                LocalVariable,
+                CalculatedPointer,
                 TemplateArgument,
                 PatternVariable,
                 GlobalVariable,

lib/include/pl/pattern_language.hpp

@@ -333,10 +333,6 @@ namespace pl {
             return this->m_defines;
         }
 
-        [[nodiscard]] const std::vector<std::shared_ptr<core::ast::ASTNode>> getAST() const {
-            return this->m_currAST;
-        }
-
         [[nodiscard]] const std::map<std::string, api::PragmaHandler>& getPragmas() const {
             return this->m_pragmas;
         }

lib/source/pl/core/lexer.cpp

@@ -52,9 +52,9 @@ namespace pl::core {
     }
 
 
-    std::optional<char> Lexer::parseCharacter() {
+    std::optional<char> Lexer::parseCharacter(bool isWindowsPath) {
         const char& c = m_sourceCode[m_cursor++];
-        if (c == '\\') {
+        if (c == '\\' && !isWindowsPath) {
             switch (m_sourceCode[m_cursor++]) {
                 case 'a':
                     return '\a';
@@ -118,15 +118,15 @@ namespace pl::core {
         return std::nullopt;
     }
 
-    std::optional<Token> Lexer::parseDirectiveValue() {
+    std::optional<Token> Lexer::parseDirectiveValue(bool forInclude) {
         std::string result;
 
         m_cursor++; // Skip space
         auto location = this->location();
 
         while (!std::isblank(m_sourceCode[m_cursor]) && !std::isspace(m_sourceCode[m_cursor]) && m_sourceCode[m_cursor] != '\0' ) {
 
-            auto character = parseCharacter();
+            auto character = parseCharacter(forInclude);
             if (!character.has_value()) {
                 return std::nullopt;
             }
@@ -176,9 +176,17 @@ namespace pl::core {
 
         while (m_sourceCode[m_cursor] != '\"') {
             char c = peek();
-            if (c == '\n' || c == '\0') {
+            if (c == '\n') {
+                m_errorLength = 1;
+                error("Unexpected newline in string literal");
+                m_line++;
+                m_lineBegin = m_cursor;
+                return std::nullopt;
+            }
+
+            if (c == '\0') {
                 m_errorLength = 1;
-                error(c == '\n' ? "Unexpected newline in string literal" : "Unexpected end of file in string literal");
+                error("Unexpected end of file in string literal");
                 return std::nullopt;
             }
 
@@ -599,15 +607,16 @@ namespace pl::core {
                 if (processToken(&Lexer::parseDirectiveName, directiveName)) {
                     Token::Directive directive = get<Token::Directive>(m_tokens.back().value);
                     if (m_line != line || directive == Token::Directive::Define || directive == Token::Directive::Undef ||
-                         peek(0) == 0  || directive == Token::Directive::IfDef  || directive == Token::Directive::IfNDef)
+                         peek(0) == 0  || directive == Token::Directive::IfDef  || directive == Token::Directive::IfNDef ||
+                         directive == Token::Directive::EndIf)
                         continue;
                     if (peek(0) == '\n') {
                         m_line++;
                         m_lineBegin = m_cursor;
                         m_cursor++;
                         continue;
                     }
-                    auto directiveValue = parseDirectiveValue();
+                    auto directiveValue = parseDirectiveValue(directive == Token::Directive::Include);
                     if (directiveValue.has_value()) {
                         addToken(directiveValue.value());
                         if (m_line != line || peek(0) == 0)
@@ -661,7 +670,7 @@ namespace pl::core {
             m_cursor++;
         }
 
-        addToken(makeToken(Separator::EndOfProgram));
+        addToken(makeToken(Separator::EndOfProgram,0));
 
         return { m_tokens, collectErrors() };
     }