Enable atomizer to wait until synched with watchtowers

Previously, the atomizer controller would make one attempt to connect to
watchtowers and would return false if the attempt failed.  This required
watchtowers to be initialized before atomizers, which required adding a
sleep to the atomizer end-to-end integration test so the watchtowers had time
to initialize.  This change enables the atomizer to make multiple attempts to
connect to watchtowers and precludes the need for the sleep in the
integration test.

Signed-off-by: Michael L. Szulczewski <[email protected]>

src/uhs/atomizer/atomizer/controller.cpp

@@ -58,11 +58,15 @@ namespace cbdc::atomizer {
     }
 
     auto controller::init() -> bool {
-        if(!m_watchtower_network.cluster_connect(
-               m_opts.m_watchtower_internal_endpoints)) {
-            m_logger->error("Failed to connect to watchtowers.");
-            return false;
+        m_watchtower_network.cluster_connect(
+            m_opts.m_watchtower_internal_endpoints,
+            false);
+        while(!m_watchtower_network.connected_to_one()) {
+            // TODO:  should we limit the number of attempts?
+            m_logger->warn("Waiting to connect to watchtowers...");
+            std::this_thread::sleep_for(std::chrono::seconds(1));
         }
+        m_logger->info("Connected to watchtower network.");
 
         auto raft_params = nuraft::raft_params();
         raft_params.election_timeout_lower_bound_

tests/integration/atomizer_end_to_end_test.cpp

@@ -47,7 +47,7 @@ class atomizer_end_to_end_test : public ::testing::Test {
             ASSERT_TRUE(m_ctl_watchtower->init());
         });
 
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        // std::this_thread::sleep_for(std::chrono::milliseconds(100));
 
         ASSERT_TRUE(m_ctl_atomizer->init());
         ASSERT_TRUE(m_ctl_archiver->init());