Feature/async-backing (#1825)

* Async-backing impl Signed-off-by: iceseer <[email protected]> * Prospective parachains impl Signed-off-by: iceseer <[email protected]> * remove validation protocol v1 Signed-off-by: iceseer <[email protected]> --------- Signed-off-by: iceseer <[email protected]> Co-authored-by: Harrm <[email protected]> Co-authored-by: kamilsa <[email protected]>
qdrvm · Feb 27, 2024 · 5183aa6 · 5183aa6
1 parent 351f5bc
commit 5183aa6
Show file tree

Hide file tree

Showing 87 changed files with 10,872 additions and 1,149 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -222,6 +222,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/kagomeConfig.cmake
 export(PACKAGE kagome)
 
 if(TESTING)
+  add_compile_definitions(CFG_TESTING)
   enable_testing()
   add_subdirectory(test)
 endif()

diff --git a/core/consensus/babe/impl/babe.cpp b/core/consensus/babe/impl/babe.cpp
@@ -31,6 +31,7 @@
 #include "parachain/availability/bitfield/store.hpp"
 #include "parachain/backing/store.hpp"
 #include "parachain/parachain_inherent_data.hpp"
+#include "parachain/validator/parachain_processor.hpp"
 #include "primitives/inherent_data.hpp"
 #include "runtime/runtime_api/offchain_worker_api.hpp"
 #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp"
@@ -74,7 +75,7 @@ namespace kagome::consensus::babe {
       std::shared_ptr<crypto::Sr25519Provider> sr25519_provider,
       std::shared_ptr<BabeBlockValidator> validating,
       std::shared_ptr<parachain::BitfieldStore> bitfield_store,
-      std::shared_ptr<parachain::BackingStore> backing_store,
+      std::shared_ptr<parachain::BackedCandidatesSource> candidates_source,
       std::shared_ptr<dispute::DisputeCoordinator> dispute_coordinator,
       std::shared_ptr<authorship::Proposer> proposer,
       primitives::events::StorageSubscriptionEnginePtr storage_sub_engine,
@@ -95,7 +96,7 @@ namespace kagome::consensus::babe {
         sr25519_provider_(std::move(sr25519_provider)),
         validating_(std::move(validating)),
         bitfield_store_(std::move(bitfield_store)),
-        backing_store_(std::move(backing_store)),
+        candidates_source_(std::move(candidates_source)),
         dispute_coordinator_(std::move(dispute_coordinator)),
         proposer_(std::move(proposer)),
         storage_sub_engine_(std::move(storage_sub_engine)),
@@ -114,7 +115,7 @@ namespace kagome::consensus::babe {
     BOOST_ASSERT(sr25519_provider_);
     BOOST_ASSERT(validating_);
     BOOST_ASSERT(bitfield_store_);
-    BOOST_ASSERT(backing_store_);
+    BOOST_ASSERT(candidates_source_);
     BOOST_ASSERT(dispute_coordinator_);
     BOOST_ASSERT(proposer_);
     BOOST_ASSERT(chain_sub_engine_);
@@ -329,9 +330,13 @@ namespace kagome::consensus::babe {
       auto &relay_parent = parent_.hash;
       parachain_inherent_data.bitfields =
           bitfield_store_->getBitfields(relay_parent);
+      SL_INFO(log_,
+              "Bitfields set for block.(count={}, relay_parent={})",
+              parachain_inherent_data.bitfields.size(),
+              relay_parent);
 
       parachain_inherent_data.backed_candidates =
-          backing_store_->get(relay_parent);
+          candidates_source_->getBackedCandidates(relay_parent);
       SL_TRACE(log_,
                "Get backed candidates from store.(count={}, relay_parent={})",
                parachain_inherent_data.backed_candidates.size(),

diff --git a/core/consensus/babe/impl/babe.hpp b/core/consensus/babe/impl/babe.hpp
@@ -57,7 +57,8 @@ namespace kagome::dispute {
 
 namespace kagome::parachain {
   class BitfieldStore;
-  class BackingStore;
+  struct ParachainProcessorImpl;
+  struct BackedCandidatesSource;
 }  // namespace kagome::parachain
 
 namespace kagome::network {
@@ -99,7 +100,7 @@ namespace kagome::consensus::babe {
         std::shared_ptr<crypto::Sr25519Provider> sr25519_provider,
         std::shared_ptr<BabeBlockValidator> validating,
         std::shared_ptr<parachain::BitfieldStore> bitfield_store,
-        std::shared_ptr<parachain::BackingStore> backing_store,
+        std::shared_ptr<parachain::BackedCandidatesSource> candidates_source,
         std::shared_ptr<dispute::DisputeCoordinator> dispute_coordinator,
         std::shared_ptr<authorship::Proposer> proposer,
         primitives::events::StorageSubscriptionEnginePtr storage_sub_engine,
@@ -157,7 +158,7 @@ namespace kagome::consensus::babe {
     std::shared_ptr<crypto::Sr25519Provider> sr25519_provider_;
     std::shared_ptr<BabeBlockValidator> validating_;
     std::shared_ptr<parachain::BitfieldStore> bitfield_store_;
-    std::shared_ptr<parachain::BackingStore> backing_store_;
+    std::shared_ptr<parachain::BackedCandidatesSource> candidates_source_;
     std::shared_ptr<dispute::DisputeCoordinator> dispute_coordinator_;
     std::shared_ptr<authorship::Proposer> proposer_;
     primitives::events::StorageSubscriptionEnginePtr storage_sub_engine_;

diff --git a/core/consensus/beefy/types.hpp b/core/consensus/beefy/types.hpp
@@ -66,8 +66,8 @@ namespace kagome::consensus::beefy {
     Commitment commitment;
     std::vector<std::optional<crypto::EcdsaSignature>> signatures;
   };
-  inline scale::ScaleEncoderStream &operator<<(scale::ScaleEncoderStream &s,
-                                               const SignedCommitment &v) {
+  inline ::scale::ScaleEncoderStream &operator<<(::scale::ScaleEncoderStream &s,
+                                                 const SignedCommitment &v) {
     s << v.commitment;
     size_t count = 0;
     common::Buffer bits;
@@ -83,16 +83,16 @@ namespace kagome::consensus::beefy {
     }
     s << bits;
     s << static_cast<uint32_t>(v.signatures.size());
-    s << scale::CompactInteger{count};
+    s << ::scale::CompactInteger{count};
     for (auto &sig : v.signatures) {
       if (sig) {
         s << *sig;
       }
     }
     return s;
   }
-  inline scale::ScaleDecoderStream &operator>>(scale::ScaleDecoderStream &s,
-                                               SignedCommitment &v) {
+  inline ::scale::ScaleDecoderStream &operator>>(::scale::ScaleDecoderStream &s,
+                                                 SignedCommitment &v) {
     s >> v.commitment;
     common::Buffer bits;
     s >> bits;
@@ -105,12 +105,12 @@ namespace kagome::consensus::beefy {
     uint32_t total = 0;
     s >> total;
     if (bits.size() * 8 < total) {
-      scale::raise(scale::DecodeError::NOT_ENOUGH_DATA);
+      ::scale::raise(::scale::DecodeError::NOT_ENOUGH_DATA);
     }
-    scale::CompactInteger actual_count;
+    ::scale::CompactInteger actual_count;
     s >> actual_count;
     if (actual_count != expected_count) {
-      scale::raise(scale::DecodeError::TOO_MANY_ITEMS);
+      ::scale::raise(::scale::DecodeError::TOO_MANY_ITEMS);
     }
     v.signatures.resize(total);
     for (size_t i = 0; i < total; ++i) {

diff --git a/core/crypto/type_hasher.hpp b/core/crypto/type_hasher.hpp
@@ -15,12 +15,8 @@ namespace kagome::crypto {
 
   template <typename H, typename... T>
   inline void hashTypes(H &hasher, common::Blob<H::kOutlen> &out, T &&...t) {
-    kagome::scale::encode(
-        [&](const uint8_t *const val, size_t count) {
-          hasher.update({val, count});
-        },
-        std::forward<T>(t)...);
-
+    auto val = ::scale::encode(std::forward<T>(t)...).value();
+    hasher.update(val);
     hasher.get_final(out);
   }
 
@@ -63,8 +59,9 @@ namespace kagome::crypto {
       }
       return *opt_hash_;
     }
-
+#ifndef CFG_TESTING
    private:
+#endif  // CFG_TESTING
     T type_;
     mutable std::optional<HashType> opt_hash_{};
   };

diff --git a/core/dispute_coordinator/impl/storage_impl.cpp b/core/dispute_coordinator/impl/storage_impl.cpp
@@ -106,7 +106,7 @@ namespace kagome::dispute {
       const StoredWindow &stored_window) {
     auto dispute_space = storage_->getSpace(storage::Space::kDisputeData);
 
-    OUTCOME_TRY(encoded, scale::encode(stored_window));
+    OUTCOME_TRY(encoded, ::scale::encode(stored_window));
 
     OUTCOME_TRY(dispute_space->put(storage::kSessionsWindowLookupKey,
                                    common::BufferView{encoded}));

diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp
@@ -8,7 +8,7 @@
 
 #define BOOST_DI_CFG_DIAGNOSTICS_LEVEL 2
 #define BOOST_DI_CFG_CTOR_LIMIT_SIZE \
-  24  // TODO(Harrm): check how it influences on compilation time
+  32  // TODO(Harrm): check how it influences on compilation time
 
 #include <rocksdb/filter_policy.h>
 #include <rocksdb/table.h>
@@ -789,6 +789,7 @@ namespace {
             di::bind<parachain::Recovery>.template to<parachain::RecoveryImpl>(),
             di::bind<parachain::BitfieldStore>.template to<parachain::BitfieldStoreImpl>(),
             di::bind<parachain::BackingStore>.template to<parachain::BackingStoreImpl>(),
+            di::bind<parachain::BackedCandidatesSource>.template to<parachain::ParachainProcessorImpl>(),
             di::bind<parachain::Pvf>.template to<parachain::PvfImpl>(),
             di::bind<network::CollationObserver>.template to<parachain::ParachainObserverImpl>(),
             di::bind<network::ValidationObserver>.template to<parachain::ParachainObserverImpl>(),

diff --git a/core/macro/endianness_utils.hpp b/core/macro/endianness_utils.hpp
@@ -7,9 +7,11 @@
 #pragma once
 
 #ifdef _MSC_VER
+#define LE_BE_SWAP16 _byteswap_ushort
 #define LE_BE_SWAP32 _byteswap_ulong
 #define LE_BE_SWAP64 _byteswap_uint64
 #else  //_MSC_VER
+#define LE_BE_SWAP16 __builtin_bswap16
 #define LE_BE_SWAP32 __builtin_bswap32
 #define LE_BE_SWAP64 __builtin_bswap64
 #endif  //_MSC_VER

diff --git a/core/network/collation_observer.hpp b/core/network/collation_observer.hpp
@@ -8,7 +8,7 @@
 
 #include <libp2p/peer/peer_id.hpp>
 
-#include "network/types/collator_messages.hpp"
+#include "network/types/collator_messages_vstaging.hpp"
 #include "primitives/common.hpp"
 
 namespace kagome::network {
@@ -21,11 +21,13 @@ namespace kagome::network {
 
     /// Handle incoming collation stream.
     virtual void onIncomingCollationStream(
-        const libp2p::peer::PeerId &peer_id) = 0;
+        const libp2p::peer::PeerId &peer_id,
+        network::CollationVersion version) = 0;
 
     /// Handle incoming collation message.
     virtual void onIncomingMessage(
         const libp2p::peer::PeerId &peer_id,
-        CollationProtocolMessage &&collation_message) = 0;
+        network::VersionedCollatorProtocolMessage &&msg) = 0;
   };
+
 }  // namespace kagome::network
diff --git a/core/network/common.hpp b/core/network/common.hpp
@@ -29,14 +29,22 @@ namespace kagome::network {
   const libp2p::peer::ProtocolName kLightProtocol = "/{}/light/2";
   const libp2p::peer::ProtocolName kCollationProtocol{"/{}/collation/1"};
   const libp2p::peer::ProtocolName kValidationProtocol{"/{}/validation/1"};
+  const libp2p::peer::ProtocolName kCollationProtocolVStaging{
+      "/{}/collation/2"};
+  const libp2p::peer::ProtocolName kValidationProtocolVStaging{
+      "/{}/validation/2"};
   const libp2p::peer::ProtocolName kReqCollationProtocol{"/{}/req_collation/1"};
+  const libp2p::peer::ProtocolName kReqCollationVStagingProtocol{
+      "/{}/req_collation/2"};
   const libp2p::peer::ProtocolName kReqPovProtocol{"/{}/req_pov/1"};
   const libp2p::peer::ProtocolName kFetchChunkProtocol{"/{}/req_chunk/1"};
   const libp2p::peer::ProtocolName kFetchAvailableDataProtocol{
       "/{}/req_available_data/1"};
   const libp2p::peer::ProtocolName kFetchStatementProtocol{
       "/{}/req_statement/1"};
   const libp2p::peer::ProtocolName kSendDisputeProtocol{"/{}/send_dispute/1"};
+  const libp2p::peer::ProtocolName kFetchAttestedCandidateProtocol{
+      "/{}/req_attested_candidate/2"};
   const libp2p::peer::ProtocolName kBeefyProtocol{"/{}/beefy/2"};
   const libp2p::peer::ProtocolName kBeefyJustificationProtocol{
       "/{}/beefy/justifications/1"};