diff --git a/lib/include/pl/core/lexer.hpp b/lib/include/pl/core/lexer.hpp index 604e4255..779d55c9 100644 --- a/lib/include/pl/core/lexer.hpp +++ b/lib/include/pl/core/lexer.hpp @@ -26,7 +26,7 @@ namespace pl::core { bool processToken(auto parserFunction, const std::string_view& identifier); Location location() override; - std::optional parseCharacter(); + std::optional parseCharacter(bool isWindowsPath=false); std::optional parseOperator(); std::optional parseSeparator(); std::optional parseOneLineComment(); @@ -40,7 +40,7 @@ namespace pl::core { std::optional parseConstant(const std::string_view &identifier); std::optional parseStringLiteral(); std::optional parseDirectiveArgument(); - std::optional parseDirectiveValue(); + std::optional parseDirectiveValue(bool forInclude=false); std::optional parseIntegerLiteral(std::string_view literal); std::optional parseFloatingPoint(std::string_view literal, char suffix); diff --git a/lib/include/pl/core/parser.hpp b/lib/include/pl/core/parser.hpp index 8c13018d..1ad5868f 100644 --- a/lib/include/pl/core/parser.hpp +++ b/lib/include/pl/core/parser.hpp @@ -149,10 +149,10 @@ namespace pl::core { hlp::safe_unique_ptr parseUnaryExpression(); hlp::safe_unique_ptr parseMultiplicativeExpression(); hlp::safe_unique_ptr parseAdditiveExpression(); - hlp::safe_unique_ptr parseShiftExpression(); - hlp::safe_unique_ptr parseBinaryAndExpression(); - hlp::safe_unique_ptr parseBinaryXorExpression(); - hlp::safe_unique_ptr parseBinaryOrExpression(bool inMatchRange); + hlp::safe_unique_ptr parseShiftExpression(bool inTemplate); + hlp::safe_unique_ptr parseBinaryAndExpression(bool inTemplate); + hlp::safe_unique_ptr parseBinaryXorExpression(bool inTemplate); + hlp::safe_unique_ptr parseBinaryOrExpression(bool inTemplate, bool inMatchRange); hlp::safe_unique_ptr parseBooleanAnd(bool inTemplate, bool inMatchRange); hlp::safe_unique_ptr parseBooleanXor(bool inTemplate, bool inMatchRange); hlp::safe_unique_ptr parseBooleanOr(bool inTemplate, bool inMatchRange); diff --git a/lib/include/pl/core/preprocessor.hpp b/lib/include/pl/core/preprocessor.hpp index 83981918..b3ea8115 100644 --- a/lib/include/pl/core/preprocessor.hpp +++ b/lib/include/pl/core/preprocessor.hpp @@ -30,35 +30,59 @@ namespace pl::core { hlp::CompileResult> preprocess(PatternLanguage *runtime, api::Source* source, bool initialRun = true); void addDefine(const std::string &name, const std::string &value = ""); + void removeDefine(const std::string &name); void addPragmaHandler(const std::string &pragmaType, const api::PragmaHandler &handler); void addDirectiveHandler(const Token::Directive &directiveType, const api::DirectiveHandler &handler); void removePragmaHandler(const std::string &pragmaType); void removeDirectiveHandler(const Token::Directive &directiveType); - void validateExcludedLocations(); - void appendExcludedLocation(const ExcludedLocation &location); + void validateOutput(); - [[nodiscard]] auto getExcludedLocations() const { + [[nodiscard]] const std::vector &getExcludedLocations() const { return m_excludedLocations; } - [[nodiscard]] auto getResult() const { - return this->m_result; + [[nodiscard]] const std::vector &getResult() { + return m_result; } [[nodiscard]] auto getOutput() const { return this->m_output; } - void setOutput(std::vector tokens) { - m_output = tokens; + void setOutput(const std::vector &tokens) { + u32 j =0; + auto tokenCount = m_result.size(); + for (auto token : tokens) { + if (auto identifier = std::get_if(&token.value); identifier != nullptr) { + if (auto type = identifier->getType(); type > Token::Identifier::IdentifierType::ScopeResolutionUnknown) { + auto location = token.location; + if (location.source->source != "") + continue; + auto line = location.line; + auto column = location.column; + while (m_result[j].location.line < line) { + if (j >= tokenCount) + break; + j++; + } + while (m_result[j].location.column < column) { + if (j >= tokenCount) + break; + j++; + } + if (auto identifier2 = std::get_if(&m_result[j].value); identifier2 != nullptr) + identifier2->setType(type); + } + } + } } - [[nodiscard]] auto getErrors() const { + [[nodiscard]] const std::vector &getErrors() const { return this->m_errors; } - void setErrors(std::vector errors) { + void setErrors(const std::vector &errors) { m_errors = errors; } @@ -70,11 +94,15 @@ namespace pl::core { return m_initialized; } + [[nodiscard]] const api::Resolver& getResolver() const { + return m_resolver; + } + void setResolver(const api::Resolver& resolvers) { m_resolver = resolvers; } - auto getNamespaces() const { + const std::vector getNamespaces() const { return m_namespaces; } @@ -85,6 +113,7 @@ namespace pl::core { bool eof(); Location location() override; void removeKey(const Token &token); + void nextLine(u32 line); // directive handlers void handleIfDef(u32 line); void handleIfNDef(u32 line); @@ -103,6 +132,7 @@ namespace pl::core { std::unordered_map m_directiveHandlers; std::unordered_map> m_defines; + std::map m_addedDefines; std::unordered_map>> m_pragmas; std::vector m_excludedLocations; diff --git a/lib/include/pl/core/token.hpp b/lib/include/pl/core/token.hpp index 030dc601..3cdeaf3e 100644 --- a/lib/include/pl/core/token.hpp +++ b/lib/include/pl/core/token.hpp @@ -179,10 +179,11 @@ namespace pl::core { Typedef, Function, UDT, + View, FunctionVariable, FunctionParameter, - PatternLocalVariable, - PatternPlacedVariable, + LocalVariable, + CalculatedPointer, TemplateArgument, PatternVariable, GlobalVariable, diff --git a/lib/include/pl/pattern_language.hpp b/lib/include/pl/pattern_language.hpp index 89afb169..65da4c70 100644 --- a/lib/include/pl/pattern_language.hpp +++ b/lib/include/pl/pattern_language.hpp @@ -333,10 +333,6 @@ namespace pl { return this->m_defines; } - [[nodiscard]] const std::vector> getAST() const { - return this->m_currAST; - } - [[nodiscard]] const std::map& getPragmas() const { return this->m_pragmas; } diff --git a/lib/source/pl/core/lexer.cpp b/lib/source/pl/core/lexer.cpp index 05bc82e5..70c4cf88 100644 --- a/lib/source/pl/core/lexer.cpp +++ b/lib/source/pl/core/lexer.cpp @@ -52,9 +52,9 @@ namespace pl::core { } - std::optional Lexer::parseCharacter() { + std::optional Lexer::parseCharacter(bool isWindowsPath) { const char& c = m_sourceCode[m_cursor++]; - if (c == '\\') { + if (c == '\\' && !isWindowsPath) { switch (m_sourceCode[m_cursor++]) { case 'a': return '\a'; @@ -118,7 +118,7 @@ namespace pl::core { return std::nullopt; } - std::optional Lexer::parseDirectiveValue() { + std::optional Lexer::parseDirectiveValue(bool forInclude) { std::string result; m_cursor++; // Skip space @@ -126,7 +126,7 @@ namespace pl::core { while (!std::isblank(m_sourceCode[m_cursor]) && !std::isspace(m_sourceCode[m_cursor]) && m_sourceCode[m_cursor] != '\0' ) { - auto character = parseCharacter(); + auto character = parseCharacter(forInclude); if (!character.has_value()) { return std::nullopt; } @@ -176,9 +176,17 @@ namespace pl::core { while (m_sourceCode[m_cursor] != '\"') { char c = peek(); - if (c == '\n' || c == '\0') { + if (c == '\n') { + m_errorLength = 1; + error("Unexpected newline in string literal"); + m_line++; + m_lineBegin = m_cursor; + return std::nullopt; + } + + if (c == '\0') { m_errorLength = 1; - error(c == '\n' ? "Unexpected newline in string literal" : "Unexpected end of file in string literal"); + error("Unexpected end of file in string literal"); return std::nullopt; } @@ -599,7 +607,8 @@ namespace pl::core { if (processToken(&Lexer::parseDirectiveName, directiveName)) { Token::Directive directive = get(m_tokens.back().value); if (m_line != line || directive == Token::Directive::Define || directive == Token::Directive::Undef || - peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef) + peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef || + directive == Token::Directive::EndIf) continue; if (peek(0) == '\n') { m_line++; @@ -607,7 +616,7 @@ namespace pl::core { m_cursor++; continue; } - auto directiveValue = parseDirectiveValue(); + auto directiveValue = parseDirectiveValue(directive == Token::Directive::Include); if (directiveValue.has_value()) { addToken(directiveValue.value()); if (m_line != line || peek(0) == 0) @@ -661,7 +670,7 @@ namespace pl::core { m_cursor++; } - addToken(makeToken(Separator::EndOfProgram)); + addToken(makeToken(Separator::EndOfProgram,0)); return { m_tokens, collectErrors() }; } diff --git a/lib/source/pl/core/parser.cpp b/lib/source/pl/core/parser.cpp index 134aa228..b4ab6c98 100644 --- a/lib/source/pl/core/parser.cpp +++ b/lib/source/pl/core/parser.cpp @@ -180,11 +180,10 @@ namespace pl::core { path.emplace_back(getValue(-1).get()); auto identifier = std::get_if(&((m_curr[-1]).value)); - if (m_currTemplateType.empty()) { - if (identifier != nullptr) + if (identifier != nullptr) { + if (m_currTemplateType.empty()) identifier->setType(Token::Identifier::IdentifierType::FunctionUnknown); - } else { - if (identifier != nullptr) + else identifier->setType(Token::Identifier::IdentifierType::MemberUnknown); } @@ -423,11 +422,14 @@ namespace pl::core { } // (parseAdditiveExpression) < >>|<< > (parseAdditiveExpression) - hlp::safe_unique_ptr Parser::parseShiftExpression() { + hlp::safe_unique_ptr Parser::parseShiftExpression(const bool inTemplate) { auto node = this->parseAdditiveExpression(); if (node == nullptr) return nullptr; + if (inTemplate && peek(tkn::Operator::BoolGreaterThan) && peek(tkn::Operator::BoolGreaterThan, 1)) + return node; + while (true) { if (sequence(tkn::Operator::BoolGreaterThan, tkn::Operator::BoolGreaterThan)) { auto other = this->parseAdditiveExpression(); @@ -450,13 +452,13 @@ namespace pl::core { } // (parseShiftExpression) & (parseShiftExpression) - hlp::safe_unique_ptr Parser::parseBinaryAndExpression() { - auto node = this->parseShiftExpression(); + hlp::safe_unique_ptr Parser::parseBinaryAndExpression(const bool inTemplate) { + auto node = this->parseShiftExpression(inTemplate); if (node == nullptr) return nullptr; while (sequence(tkn::Operator::BitAnd)) { - auto other = this->parseShiftExpression(); + auto other = this->parseShiftExpression(inTemplate); if (other == nullptr) return nullptr; @@ -467,13 +469,13 @@ namespace pl::core { } // (parseBinaryAndExpression) ^ (parseBinaryAndExpression) - hlp::safe_unique_ptr Parser::parseBinaryXorExpression() { - auto node = this->parseBinaryAndExpression(); + hlp::safe_unique_ptr Parser::parseBinaryXorExpression(const bool inTemplate) { + auto node = this->parseBinaryAndExpression(inTemplate); if (node == nullptr) return nullptr; while (sequence(tkn::Operator::BitXor)) { - auto other = this->parseBinaryAndExpression(); + auto other = this->parseBinaryAndExpression(inTemplate); if (other == nullptr) return nullptr; @@ -484,15 +486,15 @@ namespace pl::core { } // (parseBinaryXorExpression) | (parseBinaryXorExpression) - hlp::safe_unique_ptr Parser::parseBinaryOrExpression(const bool inMatchRange) { - auto node = this->parseBinaryXorExpression(); + hlp::safe_unique_ptr Parser::parseBinaryOrExpression(const bool inTemplate, const bool inMatchRange) { + auto node = this->parseBinaryXorExpression(inTemplate); if (node == nullptr) return nullptr; if (inMatchRange && peek(tkn::Operator::BitOr)) return node; while (sequence(tkn::Operator::BitOr)) { - auto other = this->parseBinaryXorExpression(); + auto other = this->parseBinaryXorExpression(inTemplate); if (other == nullptr) return nullptr; @@ -504,7 +506,7 @@ namespace pl::core { // (parseBinaryOrExpression) < >=|<=|>|< > (parseBinaryOrExpression) hlp::safe_unique_ptr Parser::parseRelationExpression(const bool inTemplate, const bool inMatchRange) { - auto node = this->parseBinaryOrExpression(inMatchRange); + auto node = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (node == nullptr) return nullptr; @@ -513,27 +515,27 @@ namespace pl::core { while (true) { if (sequence(tkn::Operator::BoolGreaterThan, tkn::Operator::Assign)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; node = create(std::move(node), std::move(other), Token::Operator::BoolGreaterThanOrEqual); } else if (sequence(tkn::Operator::BoolLessThan, tkn::Operator::Assign)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; node = create(std::move(node), std::move(other), Token::Operator::BoolLessThanOrEqual); } else if (sequence(tkn::Operator::BoolGreaterThan)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; node = create(std::move(node), std::move(other), Token::Operator::BoolGreaterThan); } else if (sequence(tkn::Operator::BoolLessThan)) { - auto other = this->parseBinaryOrExpression(inMatchRange); + auto other = this->parseBinaryOrExpression(inTemplate, inMatchRange); if (other == nullptr) return nullptr; @@ -836,8 +838,7 @@ namespace pl::core { if (sequence(tkn::Literal::Identifier)) { auto identifier = getValue(-1).get(); auto functionIdentifier = std::get_if(&((m_curr[-1]).value)); - if (functionIdentifier != nullptr) - functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + if (MATCHES(sequence(tkn::Separator::LeftBracket) && !peek(tkn::Separator::LeftBracket))) { statement = parseMemberArrayVariable(std::move(type), constant); @@ -852,8 +853,16 @@ namespace pl::core { compoundStatement.emplace_back(std::move(initStatement)); } - + if (functionIdentifier != nullptr) + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); statement = create(unwrapSafePointerVector(std::move(compoundStatement))); + } else { + if (functionIdentifier != nullptr) { + if (peek(tkn::Operator::At, 0)) + functionIdentifier->setType(Token::Identifier::IdentifierType::View); + else + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + } } } else { statement = parseMemberVariable(std::move(type), constant, identifier); @@ -868,8 +877,17 @@ namespace pl::core { compoundStatement.emplace_back(std::move(statement)); compoundStatement.emplace_back(create(identifier, std::move(expression))); } + if (functionIdentifier != nullptr) + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); statement = create(unwrapSafePointerVector(std::move(compoundStatement))); + } else { + if (functionIdentifier != nullptr) { + if (peek(tkn::Operator::At, 0)) + functionIdentifier->setType(Token::Identifier::IdentifierType::View); + else + functionIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + } } } } else { @@ -1564,7 +1582,7 @@ namespace pl::core { do { if (sequence(tkn::Literal::Identifier)) { variableName = getValue(-1).get(); - memberIdentifier = std::get_if(&((m_curr[-1]).value)); + memberIdentifier = (Token::Identifier *)std::get_if(&((m_curr[-1]).value)); if (memberIdentifier != nullptr) { if (m_currTemplateType.empty()) memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); @@ -1585,9 +1603,12 @@ namespace pl::core { auto variableName = getValue(-2).get(); - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); if (placementOffset == nullptr) @@ -1611,9 +1632,12 @@ namespace pl::core { compounds.emplace_back(create(identifier, std::move(expression))); - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); + } return create(unwrapSafePointerVector(std::move(compounds))); } else { if (constant) { @@ -1621,9 +1645,12 @@ namespace pl::core { } } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(identifier, type.unwrapUnchecked(), nullptr, nullptr, false, false, constant); } @@ -1654,9 +1681,12 @@ namespace pl::core { errorDesc("Cannot mark placed variable as 'const'.", "Variables placed in memory are always implicitly const."); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } hlp::safe_unique_ptr placementSection; hlp::safe_unique_ptr placementOffset = parseMathematicalExpression(); if (placementOffset == nullptr) @@ -1685,9 +1715,12 @@ namespace pl::core { compoundStatement.emplace_back(std::move(initStatement)); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::LocalVariable); + } return create(unwrapSafePointerVector(std::move(compoundStatement))); } else { if (constant) { @@ -1695,9 +1728,12 @@ namespace pl::core { } } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(name, type.unwrapUnchecked(), std::move(size.unwrapUnchecked()), nullptr, nullptr, constant); } @@ -1715,13 +1751,20 @@ namespace pl::core { auto expression = parseMathematicalExpression(); if (expression == nullptr) return nullptr; - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } return create(name, type.unwrapUnchecked(), std::move(sizeType), std::move(expression)); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(name, type.unwrapUnchecked(), std::move(sizeType)); } @@ -1763,14 +1806,20 @@ namespace pl::core { auto expression = parseMathematicalExpression(); if (expression == nullptr) return nullptr; - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternPlacedVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::View); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::CalculatedPointer); + } return create(name, std::move(arrayType), std::move(sizeType), std::move(expression)); } - if (memberIdentifier != nullptr) - memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); - + if (memberIdentifier != nullptr) { + if (m_currTemplateType.empty()) + memberIdentifier->setType(Token::Identifier::IdentifierType::FunctionVariable); + else + memberIdentifier->setType(Token::Identifier::IdentifierType::PatternVariable); + } return create(name, std::move(arrayType), std::move(sizeType)); } @@ -1993,7 +2042,7 @@ namespace pl::core { name = getValue(-2).get(); auto identifier = std::get_if(&((m_curr[-2]).value)); if (identifier != nullptr) - identifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); + identifier->setType(Token::Identifier::IdentifierType::LocalVariable); enumValue = parseMathematicalExpression(); if (enumValue == nullptr) @@ -2047,7 +2096,7 @@ namespace pl::core { const auto variableName = getValue(-2).get(); auto identifier = std::get_if(&((m_curr[-2]).value)); if (identifier != nullptr) - identifier->setType(Token::Identifier::IdentifierType::PatternLocalVariable); + identifier->setType(Token::Identifier::IdentifierType::LocalVariable); member = parseFunctionVariableAssignment(variableName); } else if (const auto identifierOffset = parseCompoundAssignment(tkn::Literal::Identifier); identifierOffset.has_value()) member = parseFunctionVariableCompoundAssignment(getValue(*identifierOffset).get()); @@ -2246,8 +2295,12 @@ namespace pl::core { hlp::safe_unique_ptr placementOffset, placementSection; if (sequence(tkn::Operator::At)) { - if (identifier != nullptr) - identifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + if (identifier != nullptr) { + if (m_currTemplateType.empty()) + identifier->setType(Token::Identifier::IdentifierType::View); + else + identifier->setType(Token::Identifier::IdentifierType::PlacedVariable); + } placementOffset = parseMathematicalExpression(); if (placementOffset == nullptr) return nullptr; @@ -2346,7 +2399,8 @@ namespace pl::core { auto initStatement = parseArrayInitExpression(name); if (initStatement == nullptr) return nullptr; - + if (typedefIdentifier != nullptr) + typedefIdentifier->setType(Token::Identifier::IdentifierType::GlobalVariable); compoundStatement.emplace_back(std::move(initStatement)); } diff --git a/lib/source/pl/core/preprocessor.cpp b/lib/source/pl/core/preprocessor.cpp index 248e9c8a..d3a391f0 100644 --- a/lib/source/pl/core/preprocessor.cpp +++ b/lib/source/pl/core/preprocessor.cpp @@ -30,6 +30,7 @@ namespace pl::core { Preprocessor::Preprocessor(const Preprocessor &other) : ErrorCollector(other) { this->m_defines = other.m_defines; + this->m_addedDefines = other.m_addedDefines; this->m_pragmas = other.m_pragmas; this->m_onceIncludedFiles = other.m_onceIncludedFiles; this->m_resolver = other.m_resolver; @@ -72,6 +73,14 @@ namespace pl::core { return true; } + void Preprocessor::nextLine(u32 line) { + while (!eof() && m_token->location.line == line) { + if (m_token->type == Token::Type::Comment || m_token->type == Token::Type::DocComment) + m_output.push_back(*m_token); + m_token++; + } + } + void Preprocessor::removeKey(const Token &token) { for (u32 i = 0; i < m_keys.size(); i++) { if (m_keys[i].value == token.value) { @@ -97,7 +106,7 @@ namespace pl::core { location.column = 0; m_excludedLocations.push_back({false, location}); } - m_token++; + nextLine(m_token->location.line); continue; } if(add) { @@ -106,7 +115,7 @@ namespace pl::core { if (auto *directive = std::get_if(&m_token->value);directive != nullptr && (*directive == Token::Directive::IfDef || *directive == Token::Directive::IfNDef)) depth++; - m_token++; + nextLine(m_token->location.line); } } @@ -125,7 +134,7 @@ namespace pl::core { return; } else identifier->setType(Token::Identifier::IdentifierType::Macro); - m_token++; + nextLine(line); processIfDef(m_defines.contains(identifier->get())); } @@ -138,7 +147,7 @@ namespace pl::core { return; } else identifier->setType(Token::Identifier::IdentifierType::Macro); - m_token++; + nextLine(line); processIfDef(!m_defines.contains(identifier->get())); } @@ -195,6 +204,7 @@ namespace pl::core { m_defines.erase(name); removeKey(token); } + nextLine(line); } void Preprocessor::handlePragma(u32 line) { @@ -272,6 +282,7 @@ namespace pl::core { std::ranges::copy(preprocessor.m_onceIncludedFiles.begin(), preprocessor.m_onceIncludedFiles.end(), std::inserter(this->m_onceIncludedFiles, this->m_onceIncludedFiles.begin())); std::ranges::copy(preprocessor.m_defines.begin(), preprocessor.m_defines.end(), std::inserter(this->m_defines, this->m_defines.begin())); + std::ranges::copy(preprocessor.m_addedDefines.begin(), preprocessor.m_addedDefines.end(), std::inserter(this->m_addedDefines, this->m_addedDefines.begin())); std::ranges::copy(preprocessor.m_pragmas.begin(), preprocessor.m_pragmas.end(), std::inserter(this->m_pragmas, this->m_pragmas.begin())); std::ranges::copy(preprocessor.m_keys.begin(), preprocessor.m_keys.end(), std::inserter(this->m_keys, this->m_keys.begin())); std::ranges::copy(preprocessor.m_namespaces.begin(), preprocessor.m_namespaces.end(), std::inserter(this->m_namespaces, this->m_namespaces.begin())); @@ -287,6 +298,7 @@ namespace pl::core { m_output.push_back(entry); } } + nextLine(line); } void Preprocessor::process() { @@ -301,16 +313,12 @@ namespace pl::core { } else { m_token++; handler->second(this, line); + nextLine(line); } } else if (m_token->type == Token::Type::Comment) m_token++; else { - if (auto *identifier = std::get_if(&m_token->value); - identifier != nullptr && identifier->getType() == Token::Identifier::IdentifierType::NameSpace) { - if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) - m_namespaces.push_back(identifier->get()); - } std::vector values; std::vector resultValues; values.push_back(*m_token); @@ -357,27 +365,6 @@ namespace pl::core { } } - void Preprocessor::appendExcludedLocation(const ExcludedLocation &location) { - - auto it = std::find_if(m_excludedLocations.begin(), m_excludedLocations.end(), [&location](const ExcludedLocation& el) { - return el.isExcluded == location.isExcluded; - }); - - if (it == m_excludedLocations.end()) { - m_excludedLocations.push_back(location); - } - } - - void Preprocessor::validateExcludedLocations() { - auto size = m_excludedLocations.size(); - if (size == 0) - return; - auto excludedLocations = m_excludedLocations; - m_excludedLocations.clear(); - for (auto &location : excludedLocations) - appendExcludedLocation(location); - } - void Preprocessor::validateOutput() { std::vector output = m_output; m_output.clear(); @@ -386,15 +373,42 @@ namespace pl::core { continue; m_output.push_back(token); } + + if (m_output.empty()) + m_output.push_back(Token{Token::Type::Separator, Token::Separator::EndOfProgram, Location { m_source, 1, 1, 0}}); + else if (m_output.back() != Token::Separator::EndOfProgram) { + auto location = m_output.back().location; + location.column += 1; + location.length = 0; + m_output.push_back(Token{Token::Type::Separator, Token::Separator::EndOfProgram, location}); + } } void Preprocessor::appendToNamespaces(std::vector tokens) { - for (const auto &token : tokens) { - if (auto *identifier = std::get_if(&token.value); identifier != nullptr && identifier->getType() == Token::Identifier::IdentifierType::NameSpace) - if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) - m_namespaces.push_back(identifier->get()); + for (auto token = tokens.begin(); token != tokens.end(); token++ ) { + u32 idx = 1; + if (auto *keyword = std::get_if(&token->value); keyword != nullptr && *keyword == Token::Keyword::Namespace) { + if (auto *valueType = std::get_if(&token[1].value); + valueType != nullptr && *valueType == Token::ValueType::Auto) + idx += 1; + auto *identifier = std::get_if(&token[idx].value); + while (identifier != nullptr) { + if (auto *separator = std::get_if(&token[idx].value); + separator != nullptr && *separator == Token::Separator::EndOfProgram) + break; + if (std::ranges::find(m_namespaces, identifier->get()) == m_namespaces.end()) + m_namespaces.push_back(identifier->get()); + idx += 1; + if (auto *operatorToken = std::get_if(&token[idx].value); + operatorToken == nullptr || *operatorToken != Token::Operator::ScopeResolution) + break; + idx += 1; + identifier = std::get_if(&token[idx].value); + } + } } } + hlp::CompileResult> Preprocessor::preprocess(PatternLanguage* runtime, api::Source* source, bool initialRun) { m_source = source; m_source->content = wolv::util::replaceStrings(m_source->content, "\r\n", "\n"); @@ -416,7 +430,7 @@ namespace pl::core { this->m_defines.clear(); for (const auto& [name, value] : m_runtime->getDefines()) { - addDefine(name, value); + m_defines[name] = {Token { Token::Type::String, value, {nullptr, 0, 0, 0 } } } ; } if (!source->mainSource) { @@ -429,6 +443,10 @@ namespace pl::core { } } + for (const auto& [name, value] : m_addedDefines) { + m_defines[name] = {Token { Token::Type::String, value, {nullptr, 0, 0, 0 } } } ; + } + auto [result,errors] = lexer->lex(m_source); if (result.has_value()) m_result = std::move(result.value()); @@ -443,6 +461,8 @@ namespace pl::core { while (!eof()) process(); + appendToNamespaces(m_output); + // Handle pragmas for (const auto &[type, datas] : this->m_pragmas) { for (const auto &data : datas) { @@ -465,7 +485,12 @@ namespace pl::core { } void Preprocessor::addDefine(const std::string &name, const std::string &value) { - m_defines[name] = {Token { Token::Type::String, value, {nullptr, 0, 0, 0 } } } ; + m_addedDefines[name] = value; + } + + void Preprocessor::removeDefine(const std::string &name) { + if (m_addedDefines.contains(name)) + m_addedDefines.erase(name); } void Preprocessor::addPragmaHandler(const std::string &pragmaType, const api::PragmaHandler &handler) { diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index 599c80c7..967ce288 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -121,8 +121,7 @@ namespace pl { if (ast->empty() || !ast.has_value()) return std::nullopt; - this->m_currAST = std::move(*ast); - return m_currAST; + return ast; } bool PatternLanguage::executeString(std::string code, const std::string& source, const std::map &envVars, const std::map &inVariables, bool checkResult) {