From 3831d31b272cb84d16b6a5d3d8abefeff4b9ee19 Mon Sep 17 00:00:00 2001 From: paxcut Date: Mon, 20 May 2024 11:32:11 -0700 Subject: [PATCH 01/11] feat: Last changes for syntax highlighting. Better parser support for identifier types of variable definitions only. Fixed typo in parser_manager.cpp. Parser errors were overwriting preprocessor/lexer errors. Make sure they are all added. If ast fails to be created make sure that the parsed tokens are saved before returning. Change to a more appropriate name in preprocessor and remove excluded location validation. Rearrange identifier type enumeration values to support value ordering of importance (bigger are more important types and preferred if choice can be made). Also removed types that were not identifiers. --- lib/include/pl/core/preprocessor.hpp | 2 +- lib/include/pl/core/token.hpp | 23 ++++---- lib/source/pl/core/parser.cpp | 80 ++++++++++++--------------- lib/source/pl/core/parser_manager.cpp | 5 +- lib/source/pl/core/preprocessor.cpp | 7 +-- lib/source/pl/pattern_language.cpp | 17 ++++-- 6 files changed, 63 insertions(+), 71 deletions(-) diff --git a/lib/include/pl/core/preprocessor.hpp b/lib/include/pl/core/preprocessor.hpp index 8ae2ecdd..83981918 100644 --- a/lib/include/pl/core/preprocessor.hpp +++ b/lib/include/pl/core/preprocessor.hpp @@ -78,7 +78,7 @@ namespace pl::core { return m_namespaces; } - void appendToNamespaces(std::vector namespaces); + void appendToNamespaces(std::vector tokens); private: Preprocessor(const Preprocessor &); diff --git a/lib/include/pl/core/token.hpp b/lib/include/pl/core/token.hpp index 80dc4cc5..030dc601 100644 --- a/lib/include/pl/core/token.hpp +++ b/lib/include/pl/core/token.hpp @@ -173,23 +173,20 @@ namespace pl::core { FunctionUnknown, MemberUnknown, ScopeResolutionUnknown, - UndefinedType, - PatternVariable, + Macro, + NameSpace, + Attribute, + Typedef, + Function, + UDT, + FunctionVariable, + FunctionParameter, PatternLocalVariable, PatternPlacedVariable, TemplateArgument, - FunctionVariable, - FunctionParameter, - PlacedVariable, + PatternVariable, GlobalVariable, - Function, - Macro, - NameSpace, - Typedef, - Keyword, - BuiltInType, - Attribute, - Directive + PlacedVariable }; explicit Identifier(std::string identifier="", IdentifierType identifierType = IdentifierType::Unknown) : m_identifier(std::move(identifier)), m_type(identifierType) { } diff --git a/lib/source/pl/core/parser.cpp b/lib/source/pl/core/parser.cpp index 1a0ad3d4..62fa2b6e 100644 --- a/lib/source/pl/core/parser.cpp +++ b/lib/source/pl/core/parser.cpp @@ -1293,9 +1293,7 @@ namespace pl::core { hlp::safe_unique_ptr result = nullptr; if (sequence(tkn::Literal::Identifier)) { // Custom type - auto undefinedTypeIdentifier = (Token::Identifier *)std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + result = parseCustomType(); } else if (sequence(tkn::ValueType::Any)) { // Builtin type auto type = getValue(-1); @@ -1469,7 +1467,8 @@ namespace pl::core { if (peek(tkn::Separator::Comma)) { std::vector> variables; - + if (memberIdentifier != nullptr) + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); std::string variableName = identifier; do { if (sequence(tkn::Literal::Identifier)) { @@ -1491,9 +1490,8 @@ namespace pl::core { auto variableName = getValue(-2).get(); - auto placedIdentifier = std::get_if(&((m_curr[-1]).value)); - if (placedIdentifier != nullptr) - placedIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + if (memberIdentifier != nullptr) + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); @@ -1503,8 +1501,6 @@ namespace pl::core { if (placementSection == nullptr) return nullptr; } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); return create(variableName, type.unwrapUnchecked(), std::move(placementOffset.unwrapUnchecked()), std::move(placementSection.unwrapUnchecked()), false, false, constant); } @@ -1555,9 +1551,8 @@ namespace pl::core { if (constant) error("Cannot mark placed variable as 'const'.", "Variables placed in memory are always implicitly const."); - auto typedefIdentifier = std::get_if(&((m_curr[-1]).value)); - if (typedefIdentifier != nullptr) - typedefIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + if (memberIdentifier != nullptr) + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); @@ -1568,8 +1563,6 @@ namespace pl::core { if (placementSection == nullptr) return nullptr; } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); return create(name, type.unwrapUnchecked(), std::move(size.unwrapUnchecked()), std::move(placementOffset.unwrapUnchecked()), std::move(placementSection.unwrapUnchecked()), constant); } @@ -1737,9 +1730,9 @@ namespace pl::core { // struct Identifier { <(parseMember)...> } hlp::safe_shared_ptr Parser::parseStruct() { const auto &typeName = getValue(-1).get(); - auto undefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + auto userDefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); + if (userDefinedTypeIdentifier != nullptr) + userDefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UDT); auto typeDecl = addType(typeName, create()); if(typeDecl == nullptr) @@ -1793,9 +1786,9 @@ namespace pl::core { // union Identifier { <(parseMember)...> } hlp::safe_shared_ptr Parser::parseUnion() { const auto &typeName = getValue(-1).get(); - auto undefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + auto userDefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); + if (userDefinedTypeIdentifier != nullptr) + userDefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UDT); auto typeDecl = addType(typeName, create()); if (typeDecl == nullptr) @@ -1828,9 +1821,9 @@ namespace pl::core { // enum Identifier : (parseType) { <...> } hlp::safe_shared_ptr Parser::parseEnum() { const auto typeName = getValue(-1).get(); - auto undefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + auto userDefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); + if (userDefinedTypeIdentifier != nullptr) + userDefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UDT); if (!sequence(tkn::Operator::Colon)) { error("Expected ':' after enum declaration, got {}.", getFormattedToken(0)); @@ -1944,9 +1937,9 @@ namespace pl::core { if (sequence(tkn::ValueType::Any)) { const auto typeToken = getValue(-1); if (typeToken == Token::ValueType::CustomType) { - auto undefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + auto userDefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); + if (userDefinedTypeIdentifier != nullptr) + userDefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UDT); } type = create(Token::getTypeName(typeToken), create(typeToken)); } else if (sequence(tkn::Literal::Identifier)) { @@ -1959,9 +1952,9 @@ namespace pl::core { } else { type = getCustomType(name); - auto undefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + auto userDefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); + if (userDefinedTypeIdentifier != nullptr) + userDefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UDT); if (type == nullptr) { error("Expected a variable name followed by ':', a function call or a bitfield type name, got '{}'.", name); return nullptr; @@ -2042,9 +2035,9 @@ namespace pl::core { hlp::safe_shared_ptr Parser::parseBitfield() { const std::string typeName = getValue(-1).get(); - auto undefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + auto userDefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); + if (userDefinedTypeIdentifier != nullptr) + userDefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UDT); auto typeDecl = addType(typeName, create()); if (typeDecl == nullptr) return nullptr; @@ -2078,9 +2071,9 @@ namespace pl::core { void Parser::parseForwardDeclaration() { std::string typeName = getNamespacePrefixedNames(getValue(-1).get()).back(); - auto undefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); - if (undefinedTypeIdentifier != nullptr) - undefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UndefinedType); + auto userDefinedTypeIdentifier = std::get_if(&((m_curr[-1]).value)); + if (userDefinedTypeIdentifier != nullptr) + userDefinedTypeIdentifier->setType(Token::Identifier::IdentifierType::UDT); if (this->m_types.contains(typeName)) return; @@ -2153,9 +2146,7 @@ namespace pl::core { // (parseType) Identifier[[(parseMathematicalExpression)]] @ Integer hlp::safe_unique_ptr Parser::parseArrayVariablePlacement(const hlp::safe_shared_ptr &type) { auto name = getValue(-2).get(); - auto placedIdentifier = std::get_if(&((m_curr[-2]).value)); - if (placedIdentifier != nullptr) - placedIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + auto typedefIdentifier = std::get_if(&((m_curr[-2]).value)); hlp::safe_unique_ptr size; @@ -2178,7 +2169,6 @@ namespace pl::core { if (sequence(tkn::Operator::At)) { placementOffset = parseMathematicalExpression(); - auto typedefIdentifier = std::get_if(&((m_curr[-1]).value)); if (typedefIdentifier != nullptr) typedefIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); @@ -2191,6 +2181,8 @@ namespace pl::core { return nullptr; } } + if (typedefIdentifier != nullptr) + typedefIdentifier->setType(Token::Identifier::IdentifierType::GlobalVariable); return create(name, type.unwrapUnchecked(), std::move(size.unwrapUnchecked()), std::move(placementOffset.unwrapUnchecked()), std::move(placementSection.unwrapUnchecked())); } @@ -2198,9 +2190,7 @@ namespace pl::core { // (parseType) *Identifier : (parseType) @ Integer hlp::safe_unique_ptr Parser::parsePointerVariablePlacement(const hlp::safe_shared_ptr &type) { auto name = getValue(-2).get(); - auto placedIdentifier = std::get_if(&((m_curr[-2]).value)); - if (placedIdentifier != nullptr) - placedIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + auto typedefIdentifier = std::get_if(&((m_curr[-2]).value)); auto sizeType = parseType(); if (sizeType == nullptr) @@ -2211,7 +2201,6 @@ namespace pl::core { return nullptr; } - auto typedefIdentifier = std::get_if(&((m_curr[-1]).value)); if (typedefIdentifier != nullptr) typedefIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); @@ -2233,8 +2222,6 @@ namespace pl::core { hlp::safe_unique_ptr Parser::parsePointerArrayVariablePlacement(const hlp::safe_shared_ptr &type) { auto name = getValue(-2).get(); auto placedIdentifier = std::get_if(&((m_curr[-2]).value)); - if (placedIdentifier != nullptr) - placedIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); hlp::safe_unique_ptr size; @@ -2266,7 +2253,8 @@ namespace pl::core { error("Expected '@' after pointer placement, got {}.", getFormattedToken(0)); return nullptr; } - + if (placedIdentifier != nullptr) + placedIdentifier->setType(Token::Identifier::IdentifierType::PlacedVariable); auto placementOffset = parseMathematicalExpression(); hlp::safe_unique_ptr placementSection; diff --git a/lib/source/pl/core/parser_manager.cpp b/lib/source/pl/core/parser_manager.cpp index bf729850..2c07e010 100644 --- a/lib/source/pl/core/parser_manager.cpp +++ b/lib/source/pl/core/parser_manager.cpp @@ -28,7 +28,7 @@ pl::hlp::CompileResult ParserManager::parse(api::Sour } const auto& internals = m_patternLanguage->getInternals(); - auto oldPreprpocessor = internals.preprocessor.get(); + auto oldPreprocessor = internals.preprocessor.get(); auto preprocessor = Preprocessor(); for (const auto& [name, value] : m_patternLanguage->getDefines()) { @@ -53,7 +53,8 @@ pl::hlp::CompileResult ParserManager::parse(api::Sour parser.m_aliasNamespaceString = namespacePrefix; auto result = parser.parse(tokens.value()); - oldPreprpocessor->appendToNamespaces(tokens.value()); + oldPreprocessor->appendToNamespaces(tokens.value()); + if (result.hasErrs()) return result_t::err(result.errs); diff --git a/lib/source/pl/core/preprocessor.cpp b/lib/source/pl/core/preprocessor.cpp index 84aaf5df..9d2fe85f 100644 --- a/lib/source/pl/core/preprocessor.cpp +++ b/lib/source/pl/core/preprocessor.cpp @@ -382,9 +382,9 @@ namespace pl::core { } } - void Preprocessor::appendToNamespaces(std::vector namespaces) { - for (const auto &namespaceToken : namespaces) { - if (auto *identifier = std::get_if(&namespaceToken.value); identifier != nullptr && identifier->getType() == Token::Identifier::IdentifierType::NameSpace) + void Preprocessor::appendToNamespaces(std::vector tokens) { + for (const auto &token : tokens) { + if (auto *identifier = std::get_if(&token.value); identifier != nullptr && identifier->getType() == Token::Identifier::IdentifierType::NameSpace) if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) m_namespaces.push_back(identifier->get()); } @@ -431,7 +431,6 @@ namespace pl::core { while (!eof()) process(); - validateExcludedLocations(); // Handle pragmas for (const auto &[type, datas] : this->m_pragmas) { for (const auto &data : datas) { diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index dbddfaf9..6958ab2d 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -79,6 +79,8 @@ namespace pl { // add pragmas to preprocessor for (const auto &[name, callback] : this->m_pragmas) this->m_internals.preprocessor->addPragmaHandler(name, callback); + this->m_compileErrors.clear(); + auto [tokens, preprocessorErrors] = this->m_internals.preprocessor->preprocess(this, internalSource, true); if (!preprocessorErrors.empty()) this->m_compileErrors = std::move(preprocessorErrors); @@ -93,20 +95,25 @@ namespace pl { return std::nullopt; auto [ast, parserErrors] = this->m_internals.parser->parse(tokens.value()); - if (!parserErrors.empty()) - this->m_compileErrors = std::move(parserErrors); + if (!parserErrors.empty()) { + this->m_compileErrors.insert(m_compileErrors.end(), parserErrors.begin(), parserErrors.end()); + parserErrors.clear(); + } + + this->m_internals.preprocessor->setOutput(tokens.value()); if (!ast.has_value()) return std::nullopt; if (ast->empty()) return ast; - this->m_internals.preprocessor->setOutput(tokens.value()); auto [validated, validatorErrors] = this->m_internals.validator->validate(ast.value()); wolv::util::unused(validated); - if (!validatorErrors.empty()) - this->m_compileErrors = std::move(validatorErrors); + if (!validatorErrors.empty()) { + this->m_compileErrors.insert(m_compileErrors.end(), validatorErrors.begin(), validatorErrors.end()); + validatorErrors.clear(); + } this->m_internals.preprocessor->setErrors(this->m_compileErrors); From ed41418924036455993dcc1b5ece90e7187c7ce7 Mon Sep 17 00:00:00 2001 From: paxcut Date: Fri, 24 May 2024 19:43:00 -0700 Subject: [PATCH 02/11] fix: the debugger uses locations to set the line of the text editor where the next break will occur. The value returned by getPauseLine() is the number of the line above the one it wants, but the values it may return range from zero to lines-1. When the code that's being executed cannot be seen because it is in another file, the debugger sets the line to zero which in turn is adjusted to line number by subtracting one. As negative line number locations are asserted against getPauseLine() now makes sure that the smallest number it can return is one. fix: found a bug where multi variable definitions were being label incorrectly by the parser which caused the highlighter to ignore all instances of the variables. Due to the name of the function involved it appeared that only custom type member definitions were being processed, but it appears the function variable definitions were processed by the code. In order to tell one set of variables from the other an specific node variable that only exists in one of the two provides the litmus test. --- lib/source/pl/core/evaluator.cpp | 4 ++-- lib/source/pl/core/parser.cpp | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/source/pl/core/evaluator.cpp b/lib/source/pl/core/evaluator.cpp index 1bb18521..5b762719 100644 --- a/lib/source/pl/core/evaluator.cpp +++ b/lib/source/pl/core/evaluator.cpp @@ -1054,8 +1054,8 @@ namespace pl::core { return; evaluator->handleAbort(); - - const auto line = node->getLocation().line; + auto temp = node->getLocation().line; + const auto line = temp + (temp == 0); if (evaluator->m_shouldPauseNextLine && evaluator->m_lastPauseLine != line) { evaluator->m_shouldPauseNextLine = false; evaluator->m_lastPauseLine = line; diff --git a/lib/source/pl/core/parser.cpp b/lib/source/pl/core/parser.cpp index 62fa2b6e..3bcbdad3 100644 --- a/lib/source/pl/core/parser.cpp +++ b/lib/source/pl/core/parser.cpp @@ -1467,15 +1467,23 @@ namespace pl::core { if (peek(tkn::Separator::Comma)) { std::vector> variables; - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } std::string variableName = identifier; do { if (sequence(tkn::Literal::Identifier)) { variableName = getValue(-1).get(); memberIdentifier = (Token::Identifier *)std::get_if(&((m_curr[-1]).value)); - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } } variables.emplace_back(createShared(variableName, type.unwrapUnchecked(), nullptr, nullptr, false, false, constant)); } while (sequence(tkn::Separator::Comma)); From 44bac40db9820bea8e45493c341b0f3ab4013dad Mon Sep 17 00:00:00 2001 From: paxcut Date: Thu, 1 Aug 2024 20:42:02 -0700 Subject: [PATCH 03/11] It is finally time to commit the entire syntax highlighting code and first is always the pattern language. there are also a number of fixes and improvements as listed next. fix: problem with having no spaces between ifdef nad endif directives. renamed patternLocalvariable to localVariable and patternPlacedVariable to calculatedPointer Also introduced a view type so now we have 4 struct types, 3 function types and 2 global types. fix: global arrays with new initialization syntax showed as unknown types. Syntax highlighting doesn't use the ast anymore. fix: preprocessor directives cannot have code on the same line they are except for comments. Any extra tokens located after a fully parsed directive are simply ignored. in earlier versions the highlighting code used to sets of tokens. It needed to have access to the token sequence as it is before preprocessingso tthat it contains all comments and directives. It also needed access to the sequence that was created by the parser because it contains all the identifier definitions types. Now the rwo are merged together in the preprocessor making highlighting simpler --- lib/include/pl/core/preprocessor.hpp | 38 ++++++++++++++++++++++---- lib/include/pl/core/token.hpp | 5 ++-- lib/include/pl/pattern_language.hpp | 4 --- lib/source/pl/core/lexer.cpp | 3 ++- lib/source/pl/core/parser.cpp | 36 ++++++++++++++----------- lib/source/pl/core/preprocessor.cpp | 40 +++++++++++----------------- lib/source/pl/pattern_language.cpp | 3 +-- 7 files changed, 74 insertions(+), 55 deletions(-) diff --git a/lib/include/pl/core/preprocessor.hpp b/lib/include/pl/core/preprocessor.hpp index 83981918..b139a106 100644 --- a/lib/include/pl/core/preprocessor.hpp +++ b/lib/include/pl/core/preprocessor.hpp @@ -34,16 +34,15 @@ namespace pl::core { void addDirectiveHandler(const Token::Directive &directiveType, const api::DirectiveHandler &handler); void removePragmaHandler(const std::string &pragmaType); void removeDirectiveHandler(const Token::Directive &directiveType); - void validateExcludedLocations(); - void appendExcludedLocation(const ExcludedLocation &location); + void validateOutput(); [[nodiscard]] auto getExcludedLocations() const { return m_excludedLocations; } - [[nodiscard]] auto getResult() const { - return this->m_result; + [[nodiscard]] auto getResult() { + return &m_result; } [[nodiscard]] auto getOutput() const { @@ -51,7 +50,31 @@ namespace pl::core { } void setOutput(std::vector tokens) { - m_output = tokens; + u32 j =0; + auto tokenCount = m_result.size(); + for (auto token : tokens) { + if (auto identifier = std::get_if(&token.value); identifier != nullptr) { + if (auto type = identifier->getType(); type > Token::Identifier::IdentifierType::ScopeResolutionUnknown) { + auto location = token.location; + if (location.source->source != "") + continue; + auto line = location.line; + auto column = location.column; + while (m_result[j].location.line < line) { + if (j >= tokenCount) + break; + j++; + } + while (m_result[j].location.column < column) { + if (j >= tokenCount) + break; + j++; + } + if (auto identifier2 = std::get_if(&m_result[j].value); identifier2 != nullptr) + identifier2->setType(type); + } + } + } } [[nodiscard]] auto getErrors() const { @@ -70,6 +93,10 @@ namespace pl::core { return m_initialized; } + [[nodiscard]] const api::Resolver& getResolver() const { + return m_resolver; + } + void setResolver(const api::Resolver& resolvers) { m_resolver = resolvers; } @@ -85,6 +112,7 @@ namespace pl::core { bool eof(); Location location() override; void removeKey(const Token &token); + void nextLine(u32 line); // directive handlers void handleIfDef(u32 line); void handleIfNDef(u32 line); diff --git a/lib/include/pl/core/token.hpp b/lib/include/pl/core/token.hpp index 030dc601..3cdeaf3e 100644 --- a/lib/include/pl/core/token.hpp +++ b/lib/include/pl/core/token.hpp @@ -179,10 +179,11 @@ namespace pl::core { Typedef, Function, UDT, + View, FunctionVariable, FunctionParameter, - PatternLocalVariable, - PatternPlacedVariable, + LocalVariable, + CalculatedPointer, TemplateArgument, PatternVariable, GlobalVariable, diff --git a/lib/include/pl/pattern_language.hpp b/lib/include/pl/pattern_language.hpp index 73a28cde..cd70b9a8 100644 --- a/lib/include/pl/pattern_language.hpp +++ b/lib/include/pl/pattern_language.hpp @@ -333,10 +333,6 @@ namespace pl { return this->m_defines; } - [[nodiscard]] const std::vector> getAST() const { - return this->m_currAST; - } - [[nodiscard]] const std::map& getPragmas() const { return this->m_pragmas; } diff --git a/lib/source/pl/core/lexer.cpp b/lib/source/pl/core/lexer.cpp index 05bc82e5..baaf0ecf 100644 --- a/lib/source/pl/core/lexer.cpp +++ b/lib/source/pl/core/lexer.cpp @@ -599,7 +599,8 @@ namespace pl::core { if (processToken(&Lexer::parseDirectiveName, directiveName)) { Token::Directive directive = get(m_tokens.back().value); if (m_line != line || directive == Token::Directive::Define || directive == Token::Directive::Undef || - peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef) + peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef || + directive == Token::Directive::EndIf) continue; if (peek(0) == '\n') { m_line++; diff --git a/lib/source/pl/core/parser.cpp b/lib/source/pl/core/parser.cpp index 134aa228..88cb9e3f 100644 --- a/lib/source/pl/core/parser.cpp +++ b/lib/source/pl/core/parser.cpp @@ -180,11 +180,10 @@ namespace pl::core { path.emplace_back(getValue(-1).get()); auto identifier = std::get_if(&((m_curr[-1]).value)); - if (m_currTemplateType.empty()) { - if (identifier != nullptr) + if (identifier != nullptr) { + if (m_currTemplateType.empty()) identifier->setType(Token::Identifier::IdentifierType::FunctionUnknown); - } else { - if (identifier != nullptr) + else identifier->setType(Token::Identifier::IdentifierType::MemberUnknown); } @@ -1564,7 +1563,7 @@ namespace pl::core { do { if (sequence(tkn::Literal::Identifier)) { variableName = getValue(-1).get(); - memberIdentifier = std::get_if(&((m_curr[-1]).value)); + memberIdentifier = (Token::Identifier *)std::get_if(&((m_curr[-1]).value)); if (memberIdentifier != nullptr) { if (m_currTemplateType.empty()) memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); @@ -1586,7 +1585,7 @@ namespace pl::core { auto variableName = getValue(-2).get(); if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); @@ -1612,7 +1611,7 @@ namespace pl::core { compounds.emplace_back(create(identifier, std::move(expression))); if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); + memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); return create(unwrapSafePointerVector(std::move(compounds))); } else { @@ -1655,7 +1654,7 @@ namespace pl::core { } if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); @@ -1686,7 +1685,7 @@ namespace pl::core { } if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); + memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); return create(unwrapSafePointerVector(std::move(compoundStatement))); } else { @@ -1716,7 +1715,7 @@ namespace pl::core { if (expression == nullptr) return nullptr; if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); return create(name, type.unwrapUnchecked(), std::move(sizeType), std::move(expression)); } @@ -1764,7 +1763,7 @@ namespace pl::core { if (expression == nullptr) return nullptr; if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); return create(name, std::move(arrayType), std::move(sizeType), std::move(expression)); } @@ -1993,7 +1992,7 @@ namespace pl::core { name = getValue(-2).get(); auto identifier = std::get_if(&((m_curr[-2]).value)); if (identifier != nullptr) - identifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); + identifier->setType(Token::Identifier::IdentifierType::LocalVariable); enumValue = parseMathematicalExpression(); if (enumValue == nullptr) @@ -2047,7 +2046,7 @@ namespace pl::core { const auto variableName = getValue(-2).get(); auto identifier = std::get_if(&((m_curr[-2]).value)); if (identifier != nullptr) - identifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); + identifier->setType(Token::Identifier::IdentifierType::LocalVariable); member = parseFunctionVariableAssignment(variableName); } else if (const auto identifierOffset = parseCompoundAssignment(tkn::Literal::Identifier); identifierOffset.has_value()) member = parseFunctionVariableCompoundAssignment(getValue(*identifierOffset).get()); @@ -2246,8 +2245,12 @@ namespace pl::core { hlp::safe_unique_ptr placementOffset, placementSection; if (sequence(tkn::Operator::At)) { - if (identifier != nullptr) - identifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + if (identifier != nullptr) { + if (m_currTemplateType.empty()) + identifier->setType(Token::Identifier::IdentifierType::View); + else + identifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + } placementOffset = parseMathematicalExpression(); if (placementOffset == nullptr) return nullptr; @@ -2346,7 +2349,8 @@ namespace pl::core { auto initStatement = parseArrayInitExpression(name); if (initStatement == nullptr) return nullptr; - + if (typedefIdentifier != nullptr) + typedefIdentifier->setType(Token::Identifier::IdentifierType::GlobalVariable); compoundStatement.emplace_back(std::move(initStatement)); } diff --git a/lib/source/pl/core/preprocessor.cpp b/lib/source/pl/core/preprocessor.cpp index 6ab0b10f..f2e9c128 100644 --- a/lib/source/pl/core/preprocessor.cpp +++ b/lib/source/pl/core/preprocessor.cpp @@ -70,6 +70,14 @@ namespace pl::core { return true; } + void Preprocessor::nextLine(u32 line) { + while (!eof() && m_token->location.line == line) { + if (m_token->type == Token::Type::Comment || m_token->type == Token::Type::DocComment) + m_output.push_back(*m_token); + m_token++; + } + } + void Preprocessor::removeKey(const Token &token) { for (u32 i = 0; i < m_keys.size(); i++) if (m_keys[i].value == token.value) @@ -93,7 +101,7 @@ namespace pl::core { location.column = 0; m_excludedLocations.push_back({false, location}); } - m_token++; + nextLine(m_token->location.line); continue; } if(add) { @@ -102,7 +110,7 @@ namespace pl::core { if (auto *directive = std::get_if(&m_token->value);directive != nullptr && (*directive == Token::Directive::IfDef || *directive == Token::Directive::IfNDef)) depth++; - m_token++; + nextLine(m_token->location.line); } } @@ -121,7 +129,7 @@ namespace pl::core { return; } else identifier->setType(Token::Identifier::IdentifierType::Macro); - m_token++; + nextLine(line); processIfDef(m_defines.contains(identifier->get())); } @@ -134,7 +142,7 @@ namespace pl::core { return; } else identifier->setType(Token::Identifier::IdentifierType::Macro); - m_token++; + nextLine(line); processIfDef(!m_defines.contains(identifier->get())); } @@ -191,6 +199,7 @@ namespace pl::core { m_defines.erase(name); removeKey(token); } + nextLine(line); } void Preprocessor::handlePragma(u32 line) { @@ -283,6 +292,7 @@ namespace pl::core { m_output.push_back(entry); } } + nextLine(line); } void Preprocessor::process() { @@ -297,6 +307,7 @@ namespace pl::core { } else { m_token++; handler->second(this, line); + nextLine(line); } } else if (m_token->type == Token::Type::Comment) @@ -353,27 +364,6 @@ namespace pl::core { } } - void Preprocessor::appendExcludedLocation(const ExcludedLocation &location) { - - auto it = std::find_if(m_excludedLocations.begin(), m_excludedLocations.end(), [&location](const ExcludedLocation& el) { - return el.isExcluded == location.isExcluded; - }); - - if (it == m_excludedLocations.end()) { - m_excludedLocations.push_back(location); - } - } - - void Preprocessor::validateExcludedLocations() { - auto size = m_excludedLocations.size(); - if (size == 0) - return; - auto excludedLocations = m_excludedLocations; - m_excludedLocations.clear(); - for (auto &location : excludedLocations) - appendExcludedLocation(location); - } - void Preprocessor::validateOutput() { std::vector output = m_output; m_output.clear(); diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index 6958ab2d..01561478 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -120,8 +120,7 @@ namespace pl { if (ast->empty() || !ast.has_value()) return std::nullopt; - this->m_currAST = std::move(*ast); - return m_currAST; + return ast; } bool PatternLanguage::executeString(std::string code, const std::string& source, const std::map &envVars, const std::map &inVariables, bool checkResult) { From c92e3419da27624986715e461504246f8c5dbfa5 Mon Sep 17 00:00:00 2001 From: paxcut Date: Sat, 3 Aug 2024 10:50:23 -0700 Subject: [PATCH 04/11] fix: Function variables were being labeled as member variables. Views were not being labeled properly as arrays or plain variables. fix: The end of program token was causing iterator out of bounds issues because it was assigned the default size of 1 even though it doesn't add to the size of the input so its size should reallybe zero. Note: The changes included in this PR are necessary for my ImHex open PR changes to compile, so tests wont be passed on the ImHex PR unless this or parts of it are merged and the pattern language submodule oh ImHex is updated --- lib/source/pl/core/lexer.cpp | 2 +- lib/source/pl/core/parser.cpp | 58 ++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/lib/source/pl/core/lexer.cpp b/lib/source/pl/core/lexer.cpp index baaf0ecf..a57d2f5c 100644 --- a/lib/source/pl/core/lexer.cpp +++ b/lib/source/pl/core/lexer.cpp @@ -662,7 +662,7 @@ namespace pl::core { m_cursor++; } - addToken(makeToken(Separator::EndOfProgram)); + addToken(makeToken(Separator::EndOfProgram,0)); return { m_tokens, collectErrors() }; } diff --git a/lib/source/pl/core/parser.cpp b/lib/source/pl/core/parser.cpp index 88cb9e3f..6f849a6c 100644 --- a/lib/source/pl/core/parser.cpp +++ b/lib/source/pl/core/parser.cpp @@ -835,8 +835,7 @@ namespace pl::core { if (sequence(tkn::Literal::Identifier)) { auto identifier = getValue(-1).get(); auto functionIdentifier = std::get_if(&((m_curr[-1]).value)); - if (functionIdentifier != nullptr) - functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + if (MATCHES(sequence(tkn::Separator::LeftBracket) && !peek(tkn::Separator::LeftBracket))) { statement = parseMemberArrayVariable(std::move(type), constant); @@ -851,8 +850,16 @@ namespace pl::core { compoundStatement.emplace_back(std::move(initStatement)); } - + if (functionIdentifier != nullptr) + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); statement = create(unwrapSafePointerVector(std::move(compoundStatement))); + } else { + if (functionIdentifier != nullptr) { + if (peek(tkn::Operator::At, 0)) + functionIdentifier->setType(Token::Identifier::IdentifierType::View); + else + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + } } } else { statement = parseMemberVariable(std::move(type), constant, identifier); @@ -867,8 +874,17 @@ namespace pl::core { compoundStatement.emplace_back(std::move(statement)); compoundStatement.emplace_back(create(identifier, std::move(expression))); } + if (functionIdentifier != nullptr) + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); statement = create(unwrapSafePointerVector(std::move(compoundStatement))); + } else { + if (functionIdentifier != nullptr) { + if (peek(tkn::Operator::At, 0)) + functionIdentifier->setType(Token::Identifier::IdentifierType::View); + else + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + } } } } else { @@ -1584,9 +1600,12 @@ namespace pl::core { auto variableName = getValue(-2).get(); - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); if (placementOffset == nullptr) @@ -1610,9 +1629,12 @@ namespace pl::core { compounds.emplace_back(create(identifier, std::move(expression))); - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); + } return create(unwrapSafePointerVector(std::move(compounds))); } else { if (constant) { @@ -1620,9 +1642,12 @@ namespace pl::core { } } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(identifier, type.unwrapUnchecked(), nullptr, nullptr, false, false, constant); } @@ -1653,9 +1678,12 @@ namespace pl::core { errorDesc("Cannot mark placed variable as 'const'.", "Variables placed in memory are always implicitly const."); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); if (placementOffset == nullptr) From 9085f1cb54a9a886b9d7ba128292bb4f8b2c9592 Mon Sep 17 00:00:00 2001 From: paxcut Date: Sat, 3 Aug 2024 19:38:09 -0700 Subject: [PATCH 05/11] Fix: There were still cases of function variables identified as member variables. These changes should take care of them all. --- lib/source/pl/core/parser.cpp | 53 ++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/lib/source/pl/core/parser.cpp b/lib/source/pl/core/parser.cpp index 6f849a6c..0d60ba60 100644 --- a/lib/source/pl/core/parser.cpp +++ b/lib/source/pl/core/parser.cpp @@ -1712,9 +1712,12 @@ namespace pl::core { compoundStatement.emplace_back(std::move(initStatement)); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); + } return create(unwrapSafePointerVector(std::move(compoundStatement))); } else { if (constant) { @@ -1722,9 +1725,12 @@ namespace pl::core { } } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(name, type.unwrapUnchecked(), std::move(size.unwrapUnchecked()), nullptr, nullptr, constant); } @@ -1742,13 +1748,20 @@ namespace pl::core { auto expression = parseMathematicalExpression(); if (expression == nullptr) return nullptr; - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } return create(name, type.unwrapUnchecked(), std::move(sizeType), std::move(expression)); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(name, type.unwrapUnchecked(), std::move(sizeType)); } @@ -1790,14 +1803,20 @@ namespace pl::core { auto expression = parseMathematicalExpression(); if (expression == nullptr) return nullptr; - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } return create(name, std::move(arrayType), std::move(sizeType), std::move(expression)); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(name, std::move(arrayType), std::move(sizeType)); } From 513721860660f44b4200cc5d00a9fc5282f03d1c Mon Sep 17 00:00:00 2001 From: paxcut Date: Tue, 6 Aug 2024 15:33:54 -0700 Subject: [PATCH 06/11] Fix: When a string ends prematurely with a newline, the newline wasn't increasing the number of lines so all subsequent tokens had the wrong line location. This was causing heap corruption errors in some cases when it should be causing index out of bounds errors like in the other cases which is kind of suspicious. --- lib/source/pl/core/lexer.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/source/pl/core/lexer.cpp b/lib/source/pl/core/lexer.cpp index a57d2f5c..fab9414e 100644 --- a/lib/source/pl/core/lexer.cpp +++ b/lib/source/pl/core/lexer.cpp @@ -176,9 +176,17 @@ namespace pl::core { while (m_sourceCode[m_cursor] != '\"') { char c = peek(); - if (c == '\n' || c == '\0') { + if (c == '\n') { m_errorLength = 1; - error(c == '\n' ? "Unexpected newline in string literal" : "Unexpected end of file in string literal"); + error("Unexpected newline in string literal"); + m_line++; + m_lineBegin = m_cursor; + return std::nullopt; + } + + if (c == '\0') { + m_errorLength = 1; + error("Unexpected end of file in string literal"); return std::nullopt; } From 92c24c36cdeb58d0d144669a4e1e6a8c91596cae Mon Sep 17 00:00:00 2001 From: paxcut Date: Thu, 8 Aug 2024 08:54:00 -0700 Subject: [PATCH 07/11] Fix: fix for issue 1283. If a templated template struct had a non-type argument in its last position then the resulting double closing template delimiters were being parsed as a binary shift operator. The fix extends the inclusion of the `inTemplate` bool to the shift operator so that it can return before adding the wrong result to the mathematical expression. --- lib/include/pl/core/parser.hpp | 8 ++++---- lib/source/pl/core/parser.cpp | 33 ++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/lib/include/pl/core/parser.hpp b/lib/include/pl/core/parser.hpp index 8c13018d..1ad5868f 100644 --- a/lib/include/pl/core/parser.hpp +++ b/lib/include/pl/core/parser.hpp @@ -149,10 +149,10 @@ namespace pl::core { hlp::safe_unique_ptr parseUnaryExpression(); hlp::safe_unique_ptr parseMultiplicativeExpression(); hlp::safe_unique_ptr parseAdditiveExpression(); - hlp::safe_unique_ptr parseShiftExpression(); - hlp::safe_unique_ptr parseBinaryAndExpression(); - hlp::safe_unique_ptr parseBinaryXorExpression(); - hlp::safe_unique_ptr parseBinaryOrExpression(bool inMatchRange); + hlp::safe_unique_ptr parseShiftExpression(bool inTemplate); + hlp::safe_unique_ptr parseBinaryAndExpression(bool inTemplate); + hlp::safe_unique_ptr parseBinaryXorExpression(bool inTemplate); + hlp::safe_unique_ptr parseBinaryOrExpression(bool inTemplate, bool inMatchRange); hlp::safe_unique_ptr parseBooleanAnd(bool inTemplate, bool inMatchRange); hlp::safe_unique_ptr parseBooleanXor(bool inTemplate, bool inMatchRange); hlp::safe_unique_ptr parseBooleanOr(bool inTemplate, bool inMatchRange); diff --git a/lib/source/pl/core/parser.cpp b/lib/source/pl/core/parser.cpp index 0d60ba60..b4ab6c98 100644 --- a/lib/source/pl/core/parser.cpp +++ b/lib/source/pl/core/parser.cpp @@ -422,11 +422,14 @@ namespace pl::core { } // (parseAdditiveExpression) < >>|<< > (parseAdditiveExpression) - hlp::safe_unique_ptr Parser::parseShiftExpression() { + hlp::safe_unique_ptr Parser::parseShiftExpression(const bool inTemplate) { auto node = this->parseAdditiveExpression(); if (node == nullptr) return nullptr; + if (inTemplate && peek(tkn::Operator::BoolGreaterThan) && peek(tkn::Operator::BoolGreaterThan, 1)) + return node; + while (true) { if (sequence(tkn::Operator::BoolGreaterThan, tkn::Operator::BoolGreaterThan)) { auto other = this->parseAdditiveExpression(); @@ -449,13 +452,13 @@ namespace pl::core { } // (parseShiftExpression) & (parseShiftExpression) - hlp::safe_unique_ptr Parser::parseBinaryAndExpression() { - auto node = this->parseShiftExpression(); + hlp::safe_unique_ptr Parser::parseBinaryAndExpression(const bool inTemplate) { + auto node = this->parseShiftExpression(inTemplate); if (node == nullptr) return nullptr; while (sequence(tkn::Operator::BitAnd)) { - auto other = this->parseShiftExpression(); + auto other = this->parseShiftExpression(inTemplate); if (other == nullptr) return nullptr; @@ -466,13 +469,13 @@ namespace pl::core { } // (parseBinaryAndExpression) ^ (parseBinaryAndExpression) - hlp::safe_unique_ptr Parser::parseBinaryXorExpression() { - auto node = this->parseBinaryAndExpression(); + hlp::safe_unique_ptr Parser::parseBinaryXorExpression(const bool inTemplate) { + auto node = this->parseBinaryAndExpression(inTemplate); if (node == nullptr) return nullptr; while (sequence(tkn::Operator::BitXor)) { - auto other = this->parseBinaryAndExpression(); + auto other = this->parseBinaryAndExpression(inTemplate); if (other == nullptr) return nullptr; @@ -483,15 +486,15 @@ namespace pl::core { } // (parseBinaryXorExpression) | (parseBinaryXorExpression) - hlp::safe_unique_ptr Parser::parseBinaryOrExpression(const bool inMatchRange) { - auto node = this->parseBinaryXorExpression(); + hlp::safe_unique_ptr Parser::parseBinaryOrExpression(const bool inTemplate, const bool inMatchRange) { + auto node = this->parseBinaryXorExpression(inTemplate); if (node == nullptr) return nullptr; if (inMatchRange && peek(tkn::Operator::BitOr)) return node; while (sequence(tkn::Operator::BitOr)) { - auto other = this->parseBinaryXorExpression(); + auto other = this->parseBinaryXorExpression(inTemplate); if (other == nullptr) return nullptr; @@ -503,7 +506,7 @@ namespace pl::core { // (parseBinaryOrExpression) < >=|<=|>|< > (parseBinaryOrExpression) hlp::safe_unique_ptr Parser::parseRelationExpression(const bool inTemplate, const bool inMatchRange) { - auto node = this->parseBinaryOrExpression(inMatchRange); + auto node = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (node == nullptr) return nullptr; @@ -512,27 +515,27 @@ namespace pl::core { while (true) { if (sequence(tkn::Operator::BoolGreaterThan, tkn::Operator::Assign)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; node = create(std::move(node), std::move(other), Token::Operator::BoolGreaterThanOrEqual); } else if (sequence(tkn::Operator::BoolLessThan, tkn::Operator::Assign)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; node = create(std::move(node), std::move(other), Token::Operator::BoolLessThanOrEqual); } else if (sequence(tkn::Operator::BoolGreaterThan)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; node = create(std::move(node), std::move(other), Token::Operator::BoolGreaterThan); } else if (sequence(tkn::Operator::BoolLessThan)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; From 1154aa37e824b4983013c7944981ee509944021a Mon Sep 17 00:00:00 2001 From: paxcut Date: Sat, 10 Aug 2024 08:53:24 -0700 Subject: [PATCH 08/11] Fix: If last statement is a directive or a comment then the end of program token gets removed from token sequence when the directive or comment are removed which results in "unexpected end of file reached" error. Fixed making sure the token gets added if it is missing during token sequence validation prior to returning it. --- lib/source/pl/core/preprocessor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/source/pl/core/preprocessor.cpp b/lib/source/pl/core/preprocessor.cpp index f2e9c128..99693b04 100644 --- a/lib/source/pl/core/preprocessor.cpp +++ b/lib/source/pl/core/preprocessor.cpp @@ -372,6 +372,12 @@ namespace pl::core { continue; m_output.push_back(token); } + if (!m_output.empty() && m_output.back() != Token::Separator::EndOfProgram ) { + auto location = m_output.back().location; + location.column += 1; + location.length = 0; + m_output.push_back(Token{Token::Type::Separator, Token::Separator::EndOfProgram, location}); + } } void Preprocessor::appendToNamespaces(std::vector tokens) { From 2173dde3265bf5abc11655fec85f076934b03df7 Mon Sep 17 00:00:00 2001 From: paxcut Date: Mon, 12 Aug 2024 00:22:38 -0700 Subject: [PATCH 09/11] Fix: As originally coded the IMPORTED mechanic created to select out code if file was imported could not be made to work. Using the addDefine in Preprocessor never worked so an addDefine was added in PatternLanguage to store system defines but that would add the defines to all the preprocessors and not just the imported ones. The fix consists on creating a new variable in Preprocessor to store defines that come from Preprocessor::addDefine which are not deleted on first run and inserted into the variable that stores the defines on every run. improv: turned function return values and arguments into const references to avoid copies --- lib/include/pl/core/preprocessor.hpp | 16 +++++++++------- lib/source/pl/core/preprocessor.cpp | 15 +++++++++++++-- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/include/pl/core/preprocessor.hpp b/lib/include/pl/core/preprocessor.hpp index b139a106..b3ea8115 100644 --- a/lib/include/pl/core/preprocessor.hpp +++ b/lib/include/pl/core/preprocessor.hpp @@ -30,6 +30,7 @@ namespace pl::core { hlp::CompileResult> preprocess(PatternLanguage *runtime, api::Source* source, bool initialRun = true); void addDefine(const std::string &name, const std::string &value = ""); + void removeDefine(const std::string &name); void addPragmaHandler(const std::string &pragmaType, const api::PragmaHandler &handler); void addDirectiveHandler(const Token::Directive &directiveType, const api::DirectiveHandler &handler); void removePragmaHandler(const std::string &pragmaType); @@ -37,19 +38,19 @@ namespace pl::core { void validateOutput(); - [[nodiscard]] auto getExcludedLocations() const { + [[nodiscard]] const std::vector &getExcludedLocations() const { return m_excludedLocations; } - [[nodiscard]] auto getResult() { - return &m_result; + [[nodiscard]] const std::vector &getResult() { + return m_result; } [[nodiscard]] auto getOutput() const { return this->m_output; } - void setOutput(std::vector tokens) { + void setOutput(const std::vector &tokens) { u32 j =0; auto tokenCount = m_result.size(); for (auto token : tokens) { @@ -77,11 +78,11 @@ namespace pl::core { } } - [[nodiscard]] auto getErrors() const { + [[nodiscard]] const std::vector &getErrors() const { return this->m_errors; } - void setErrors(std::vector errors) { + void setErrors(const std::vector &errors) { m_errors = errors; } @@ -101,7 +102,7 @@ namespace pl::core { m_resolver = resolvers; } - auto getNamespaces() const { + const std::vector getNamespaces() const { return m_namespaces; } @@ -131,6 +132,7 @@ namespace pl::core { std::unordered_map m_directiveHandlers; std::unordered_map> m_defines; + std::map m_addedDefines; std::unordered_map>> m_pragmas; std::vector m_excludedLocations; diff --git a/lib/source/pl/core/preprocessor.cpp b/lib/source/pl/core/preprocessor.cpp index 99693b04..b7e89492 100644 --- a/lib/source/pl/core/preprocessor.cpp +++ b/lib/source/pl/core/preprocessor.cpp @@ -30,6 +30,7 @@ namespace pl::core { Preprocessor::Preprocessor(const Preprocessor &other) : ErrorCollector(other) { this->m_defines = other.m_defines; + this->m_addedDefines = other.m_addedDefines; this->m_pragmas = other.m_pragmas; this->m_onceIncludedFiles = other.m_onceIncludedFiles; this->m_resolver = other.m_resolver; @@ -277,6 +278,7 @@ namespace pl::core { std::ranges::copy(preprocessor.m_onceIncludedFiles.begin(), preprocessor.m_onceIncludedFiles.end(), std::inserter(this->m_onceIncludedFiles, this->m_onceIncludedFiles.begin())); std::ranges::copy(preprocessor.m_defines.begin(), preprocessor.m_defines.end(), std::inserter(this->m_defines, this->m_defines.begin())); + std::ranges::copy(preprocessor.m_addedDefines.begin(), preprocessor.m_addedDefines.end(), std::inserter(this->m_addedDefines, this->m_addedDefines.begin())); std::ranges::copy(preprocessor.m_pragmas.begin(), preprocessor.m_pragmas.end(), std::inserter(this->m_pragmas, this->m_pragmas.begin())); std::ranges::copy(preprocessor.m_keys.begin(), preprocessor.m_keys.end(), std::inserter(this->m_keys, this->m_keys.begin())); std::ranges::copy(preprocessor.m_namespaces.begin(), preprocessor.m_namespaces.end(), std::inserter(this->m_namespaces, this->m_namespaces.begin())); @@ -408,13 +410,17 @@ namespace pl::core { this->m_onlyIncludeOnce = false; this->m_pragmas.clear(); for (const auto& [name, value] : m_runtime->getDefines()) { - addDefine(name, value); + m_defines[name] = {Token { Token::Type::String, value, {nullptr, 0, 0, 0 } } } ; } for (const auto& [name, handler]: m_runtime->getPragmas()) { addPragmaHandler(name, handler); } } + for (const auto& [name, value] : m_addedDefines) { + m_defines[name] = {Token { Token::Type::String, value, {nullptr, 0, 0, 0 } } } ; + } + auto [result,errors] = lexer->lex(m_source); if (result.has_value()) m_result = std::move(result.value()); @@ -451,7 +457,12 @@ namespace pl::core { } void Preprocessor::addDefine(const std::string &name, const std::string &value) { - m_defines[name] = {Token { Token::Type::String, value, {nullptr, 0, 0, 0 } } } ; + m_addedDefines[name] = value; + } + + void Preprocessor::removeDefine(const std::string &name) { + if (m_addedDefines.contains(name)) + m_addedDefines.erase(name); } void Preprocessor::addPragmaHandler(const std::string &pragmaType, const api::PragmaHandler &handler) { From 1a897e36c5561c1661cdf398acef179bb3bd3634 Mon Sep 17 00:00:00 2001 From: paxcut Date: Fri, 16 Aug 2024 03:01:43 -0700 Subject: [PATCH 10/11] fix: if a windows style path delimiter was used in an include directive, the code would silently ignore the include directive. fix: In order to determine if a namespace is valid to correctly highlight it, the preprocessors compile lists of encountered namespaces that the text highlighter can use when testing newly found identifiers. This works fine as long as the preprocessors use a valid way to determine if an identifier is a namespace or not and they weren't. --- lib/include/pl/core/lexer.hpp | 4 ++-- lib/source/pl/core/lexer.cpp | 10 +++++----- lib/source/pl/core/preprocessor.cpp | 27 ++++++++++++++++++++++----- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/lib/include/pl/core/lexer.hpp b/lib/include/pl/core/lexer.hpp index 604e4255..779d55c9 100644 --- a/lib/include/pl/core/lexer.hpp +++ b/lib/include/pl/core/lexer.hpp @@ -26,7 +26,7 @@ namespace pl::core { bool processToken(auto parserFunction, const std::string_view& identifier); Location location() override; - std::optional parseCharacter(); + std::optional parseCharacter(bool isWindowsPath=false); std::optional parseOperator(); std::optional parseSeparator(); std::optional parseOneLineComment(); @@ -40,7 +40,7 @@ namespace pl::core { std::optional parseConstant(const std::string_view &identifier); std::optional parseStringLiteral(); std::optional parseDirectiveArgument(); - std::optional parseDirectiveValue(); + std::optional parseDirectiveValue(bool forInclude=false); std::optional parseIntegerLiteral(std::string_view literal); std::optional parseFloatingPoint(std::string_view literal, char suffix); diff --git a/lib/source/pl/core/lexer.cpp b/lib/source/pl/core/lexer.cpp index fab9414e..70c4cf88 100644 --- a/lib/source/pl/core/lexer.cpp +++ b/lib/source/pl/core/lexer.cpp @@ -52,9 +52,9 @@ namespace pl::core { } - std::optional Lexer::parseCharacter() { + std::optional Lexer::parseCharacter(bool isWindowsPath) { const char& c = m_sourceCode[m_cursor++]; - if (c == '\\') { + if (c == '\\' && !isWindowsPath) { switch (m_sourceCode[m_cursor++]) { case 'a': return '\a'; @@ -118,7 +118,7 @@ namespace pl::core { return std::nullopt; } - std::optional Lexer::parseDirectiveValue() { + std::optional Lexer::parseDirectiveValue(bool forInclude) { std::string result; m_cursor++; // Skip space @@ -126,7 +126,7 @@ namespace pl::core { while (!std::isblank(m_sourceCode[m_cursor]) && !std::isspace(m_sourceCode[m_cursor]) && m_sourceCode[m_cursor] != '\0' ) { - auto character = parseCharacter(); + auto character = parseCharacter(forInclude); if (!character.has_value()) { return std::nullopt; } @@ -616,7 +616,7 @@ namespace pl::core { m_cursor++; continue; } - auto directiveValue = parseDirectiveValue(); + auto directiveValue = parseDirectiveValue(directive == Token::Directive::Include); if (directiveValue.has_value()) { addToken(directiveValue.value()); if (m_line != line || peek(0) == 0) diff --git a/lib/source/pl/core/preprocessor.cpp b/lib/source/pl/core/preprocessor.cpp index b7e89492..ca5df3b9 100644 --- a/lib/source/pl/core/preprocessor.cpp +++ b/lib/source/pl/core/preprocessor.cpp @@ -315,11 +315,25 @@ namespace pl::core { } else if (m_token->type == Token::Type::Comment) m_token++; else { - if (auto *identifier = std::get_if(&m_token->value); - identifier != nullptr && identifier->getType() == Token::Identifier::IdentifierType::NameSpace) { - if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) - m_namespaces.push_back(identifier->get()); + u32 idx = 1; + if (auto *keyword = std::get_if(&m_token->value); keyword != nullptr && *keyword == Token::Keyword::Namespace) { + if (auto *valueType = std::get_if(&m_token[1].value); valueType != nullptr && *valueType == Token::ValueType::Auto) + idx += 1; + + auto *identifier = std::get_if(&m_token[idx].value); + while ( identifier != nullptr) { + if (auto *separator = std::get_if(&m_token[idx].value); separator != nullptr && *separator == Token::Separator::EndOfProgram) + break; + if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) + m_namespaces.push_back(identifier->get()); + idx += 1; + if (auto *operatorToken = std::get_if(&m_token[idx].value); operatorToken == nullptr || *operatorToken != Token::Operator::ScopeResolution) + break; + idx += 1; + identifier = std::get_if(&m_token[idx].value); + } } + std::vector values; std::vector resultValues; values.push_back(*m_token); @@ -374,7 +388,10 @@ namespace pl::core { continue; m_output.push_back(token); } - if (!m_output.empty() && m_output.back() != Token::Separator::EndOfProgram ) { + + if (m_output.empty()) + m_output.push_back(Token{Token::Type::Separator, Token::Separator::EndOfProgram, Location { m_source, 1, 1, 0}}); + else if (m_output.back() != Token::Separator::EndOfProgram) { auto location = m_output.back().location; location.column += 1; location.length = 0; From b3d13f313a67eb6d91b099273639cadf0b910438 Mon Sep 17 00:00:00 2001 From: paxcut Date: Fri, 16 Aug 2024 14:36:32 -0700 Subject: [PATCH 11/11] fix: the previous fix to help highlight namespaces was not applied to namespaces created within imported files. That uses a function and the function had the same incorrect test for namespace identification. Once the function was updated then it gets used everywhere to avoid code duplication. --- lib/source/pl/core/preprocessor.cpp | 47 +++++++++++++++-------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/lib/source/pl/core/preprocessor.cpp b/lib/source/pl/core/preprocessor.cpp index ca5df3b9..99a9fca8 100644 --- a/lib/source/pl/core/preprocessor.cpp +++ b/lib/source/pl/core/preprocessor.cpp @@ -315,25 +315,6 @@ namespace pl::core { } else if (m_token->type == Token::Type::Comment) m_token++; else { - u32 idx = 1; - if (auto *keyword = std::get_if(&m_token->value); keyword != nullptr && *keyword == Token::Keyword::Namespace) { - if (auto *valueType = std::get_if(&m_token[1].value); valueType != nullptr && *valueType == Token::ValueType::Auto) - idx += 1; - - auto *identifier = std::get_if(&m_token[idx].value); - while ( identifier != nullptr) { - if (auto *separator = std::get_if(&m_token[idx].value); separator != nullptr && *separator == Token::Separator::EndOfProgram) - break; - if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) - m_namespaces.push_back(identifier->get()); - idx += 1; - if (auto *operatorToken = std::get_if(&m_token[idx].value); operatorToken == nullptr || *operatorToken != Token::Operator::ScopeResolution) - break; - idx += 1; - identifier = std::get_if(&m_token[idx].value); - } - } - std::vector values; std::vector resultValues; values.push_back(*m_token); @@ -400,12 +381,30 @@ namespace pl::core { } void Preprocessor::appendToNamespaces(std::vector tokens) { - for (const auto &token : tokens) { - if (auto *identifier = std::get_if(&token.value); identifier != nullptr && identifier->getType() == Token::Identifier::IdentifierType::NameSpace) - if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) - m_namespaces.push_back(identifier->get()); + for (auto token = tokens.begin(); token != tokens.end(); token++ ) { + u32 idx = 1; + if (auto *keyword = std::get_if(&token->value); keyword != nullptr && *keyword == Token::Keyword::Namespace) { + if (auto *valueType = std::get_if(&token[1].value); + valueType != nullptr && *valueType == Token::ValueType::Auto) + idx += 1; + auto *identifier = std::get_if(&token[idx].value); + while (identifier != nullptr) { + if (auto *separator = std::get_if(&token[idx].value); + separator != nullptr && *separator == Token::Separator::EndOfProgram) + break; + if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) + m_namespaces.push_back(identifier->get()); + idx += 1; + if (auto *operatorToken = std::get_if(&token[idx].value); + operatorToken == nullptr || *operatorToken != Token::Operator::ScopeResolution) + break; + idx += 1; + identifier = std::get_if(&token[idx].value); + } + } } } + hlp::CompileResult> Preprocessor::preprocess(PatternLanguage* runtime, api::Source* source, bool initialRun) { m_source = source; m_source->content = wolv::util::replaceStrings(m_source->content, "\r\n", "\n"); @@ -452,6 +451,8 @@ namespace pl::core { while (!eof()) process(); + appendToNamespaces(m_output); + // Handle pragmas for (const auto &[type, datas] : this->m_pragmas) { for (const auto &data : datas) {