feat(replica): parameterize-fileddatatype

apache · Dec 4, 2023 · 2bfe595 · 2bfe595
1 parent 5dde026
commit 2bfe595
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 51 deletions.
diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp
@@ -56,7 +56,6 @@
 #include "runtime/task/task_code.h"
 #include "runtime/task/task_spec.h"
 #include "split/replica_split_manager.h"
-#include "utils/env.h"
 #include "utils/filesystem.h"
 #include "utils/fmt_logging.h"
 #include "utils/latency_tracer.h"
@@ -593,7 +592,7 @@ error_code replica::store_app_info(app_info &info, const std::string &path)
 {
     replica_app_info new_info((app_info *)&info);
     const auto &info_path = path.empty() ? utils::filesystem::path_combine(_dir, kAppInfo) : path;
-    auto err = new_info.store(info_path, dsn::utils::FileDataType::kSensitive);
+    auto err = new_info.store(info_path);
     if (dsn_unlikely(err != ERR_OK)) {
         LOG_ERROR_PREFIX("failed to save app_info to {}, error = {}", info_path, err);
     }

diff --git a/src/replica/replica_disk_migrator.cpp b/src/replica/replica_disk_migrator.cpp
@@ -31,7 +31,6 @@
 #include "replica/replication_app_base.h"
 #include "replica_disk_migrator.h"
 #include "runtime/task/async_calls.h"
-#include "utils/env.h"
 #include "utils/error_code.h"
 #include "utils/fail_point.h"
 #include "utils/filesystem.h"
@@ -252,7 +251,7 @@ bool replica_disk_migrator::migrate_replica_app_info(const replica_disk_migrate_
         return false;
     });
     replica_init_info init_info = _replica->get_app()->init_info();
-    const auto &store_init_info_err = init_info.store(_target_replica_dir, dsn::utils::FileDataType::kSensitive);
+    const auto &store_init_info_err = init_info.store(_target_replica_dir);
     if (store_init_info_err != ERR_OK) {
         LOG_ERROR_PREFIX("disk migration(origin={}, target={}) stores app init info failed({})",
                          req.origin_disk,

diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp
@@ -72,7 +72,6 @@
 #include "runtime/task/async_calls.h"
 #include "split/replica_split_manager.h"
 #include "utils/command_manager.h"
-#include "utils/env.h"
 #include "utils/filesystem.h"
 #include "utils/fmt_logging.h"
 #include "utils/ports.h"
@@ -2327,7 +2326,7 @@ replica *replica_stub::load_replica(dir_node *dn, const char *dir)
     dsn::app_info info;
     replica_app_info info2(&info);
     std::string path = utils::filesystem::path_combine(dir, replica::kAppInfo);
-    auto err = info2.load(path, dsn::utils::FileDataType::kSensitive);
+    auto err = info2.load(path);
     if (ERR_OK != err) {
         LOG_ERROR("load app-info from {} failed, err = {}", path, err);
         return nullptr;

diff --git a/src/replica/replication_app_base.cpp b/src/replica/replication_app_base.cpp
@@ -65,30 +65,7 @@ namespace replication {
 
 const std::string replica_init_info::kInitInfo = ".init-info";
 
-namespace {
-error_code write_blob_to_file(const std::string &fname, const blob &data, const dsn::utils::FileDataType &fileDataType)
-{
-    // TODO(yingchun): consider not encrypt the meta files.
-    std::string tmp_fname = fname + ".tmp";
-    auto cleanup = defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); });
-    auto s =
-        rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(fileDataType),
-                                   rocksdb::Slice(data.data(), data.length()),
-                                   tmp_fname,
-                                   /* should_sync */ true);
-    LOG_AND_RETURN_NOT_TRUE(
-        ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "write file {} failed", tmp_fname);
-    LOG_AND_RETURN_NOT_TRUE(ERROR,
-                            utils::filesystem::rename_path(tmp_fname, fname),
-                            ERR_FILE_OPERATION_FAILED,
-                            "move file from {} to {} failed",
-                            tmp_fname,
-                            fname);
-    return ERR_OK;
-}
-} // namespace
-
-error_code replica_init_info::load(const std::string &dir, const dsn::utils::FileDataType &fileDataType)
+error_code replica_init_info::load(const std::string &dir)
 {
     std::string info_path = utils::filesystem::path_combine(dir, kInitInfo);
     LOG_AND_RETURN_NOT_TRUE(ERROR,
@@ -97,17 +74,17 @@ error_code replica_init_info::load(const std::string &dir, const dsn::utils::Fil
                             "file({}) not exist",
                             info_path);
     LOG_AND_RETURN_NOT_OK(
-        ERROR, load_json(info_path, fileDataType), "load replica_init_info from {} failed", info_path);
+        ERROR, load_json(info_path), "load replica_init_info from {} failed", info_path);
     LOG_INFO("load replica_init_info from {} succeed: {}", info_path, to_string());
     return ERR_OK;
 }
 
-error_code replica_init_info::store(const std::string &dir, const dsn::utils::FileDataType &fileDataType)
+error_code replica_init_info::store(const std::string &dir)
 {
     uint64_t start = dsn_now_ns();
     std::string info_path = utils::filesystem::path_combine(dir, kInitInfo);
     LOG_AND_RETURN_NOT_OK(ERROR,
-                          store_json(info_path, fileDataType),
+                          store_json(info_path),
                           "store replica_init_info to {} failed, time_used_ns = {}",
                           info_path,
                           dsn_now_ns() - start);
@@ -118,11 +95,11 @@ error_code replica_init_info::store(const std::string &dir, const dsn::utils::Fi
     return ERR_OK;
 }
 
-error_code replica_init_info::load_json(const std::string &fname, const dsn::utils::FileDataType &fileDataType)
+error_code replica_init_info::load_json(const std::string &fname)
 {
     std::string data;
     auto s = rocksdb::ReadFileToString(
-        dsn::utils::PegasusEnv(fileDataType), fname, &data);
+        dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data);
     LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname);
     LOG_AND_RETURN_NOT_TRUE(ERROR,
                             json::json_forwarder<replica_init_info>::decode(
@@ -133,9 +110,9 @@ error_code replica_init_info::load_json(const std::string &fname, const dsn::uti
     return ERR_OK;
 }
 
-error_code replica_init_info::store_json(const std::string &fname, const dsn::utils::FileDataType &fileDataType)
+error_code replica_init_info::store_json(const std::string &fname)
 {
-    return write_blob_to_file(fname, json::json_forwarder<replica_init_info>::encode(*this), fileDataType);
+    return write_blob_to_file(fname, json::json_forwarder<replica_init_info>::encode(*this), dsn::utils::FileDataType &kSensitive);
 }
 
 std::string replica_init_info::to_string()
@@ -148,11 +125,11 @@ std::string replica_init_info::to_string()
     return oss.str();
 }
 
-error_code replica_app_info::load(const std::string &fname, const dsn::utils::FileDataType &fileDataType)
+error_code replica_app_info::load(const std::string &fname)
 {
     std::string data;
     auto s = rocksdb::ReadFileToString(
-        dsn::utils::PegasusEnv(fileDataType), fname, &data);
+        dsn::utils::PegasusEnv(dsn::utils::FileDataType::kSensitive), fname, &data);
     LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname);
     binary_reader reader(blob::create_from_bytes(std::move(data)));
     int magic = 0;
@@ -163,7 +140,7 @@ error_code replica_app_info::load(const std::string &fname, const dsn::utils::Fi
     return ERR_OK;
 }
 
-error_code replica_app_info::store(const std::string &fname, const dsn::utils::FileDataType &fileDataType)
+error_code replica_app_info::store(const std::string &fname)
 {
     binary_writer writer;
     int magic = 0xdeadbeef;
@@ -183,7 +160,7 @@ error_code replica_app_info::store(const std::string &fname, const dsn::utils::F
         marshall(writer, tmp, DSF_THRIFT_JSON);
     }
 
-    return write_blob_to_file(fname, writer.get_buffer(), fileDataType);
+    return write_blob_to_file(fname, writer.get_buffer(), dsn::utils::FileDataType::kSensitive);
 }
 
 /*static*/
@@ -241,7 +218,7 @@ error_code replication_app_base::open_internal(replica *r)
 
     _last_committed_decree = last_durable_decree();
 
-    auto err = _info.load(r->dir(), dsn::utils::FileDataType::kSensitive);
+    auto err = _info.load(r->dir());
     LOG_AND_RETURN_NOT_OK(ERROR_PREFIX, err, "load replica_init_info failed");
 
     LOG_AND_RETURN_NOT_TRUE(ERROR_PREFIX,
@@ -458,7 +435,7 @@ error_code replication_app_base::update_init_info(replica *r,
     _info.init_offset_in_shared_log = shared_log_offset;
     _info.init_offset_in_private_log = private_log_offset;
 
-    LOG_AND_RETURN_NOT_OK(ERROR_PREFIX, _info.store(r->dir(), dsn::utils::FileDataType::kSensitive), "store replica_init_info failed");
+    LOG_AND_RETURN_NOT_OK(ERROR_PREFIX, _info.store(r->dir()), "store replica_init_info failed");
 
     return ERR_OK;
 }

diff --git a/src/replica/replication_app_base.h b/src/replica/replication_app_base.h
@@ -54,6 +54,29 @@ class learn_state;
 class mutation;
 class replica;
 
+namespace{
+template <class T>
+error_code write_blob_to_file(const std::string &fname, const T &data, const dsn::utils::FileDataType &fileDataType)
+{
+    std::string tmp_fname = fname + ".tmp";
+    auto cleanup = defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); });
+    auto s =
+        rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(fileDataType),
+                                   rocksdb::Slice(data.data(), data.length()),
+                                   tmp_fname,
+                                   /* should_sync */ true);
+    LOG_AND_RETURN_NOT_TRUE(
+        ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "write file {} failed", tmp_fname);
+    LOG_AND_RETURN_NOT_TRUE(ERROR,
+                            utils::filesystem::rename_path(tmp_fname, fname),
+                            ERR_FILE_OPERATION_FAILED,
+                            "move file from {} to {} failed",
+                            tmp_fname,
+                            fname);
+    return ERR_OK;
+}
+} // namespace
+
 class replica_init_info
 {
 public:
@@ -72,13 +95,13 @@ class replica_init_info
 
 public:
     replica_init_info() { memset((void *)this, 0, sizeof(*this)); }
-    error_code load(const std::string &dir, const dsn::utils::FileDataType &fileDataType) WARN_UNUSED_RESULT;
-    error_code store(const std::string &dir, const dsn::utils::FileDataType &fileDataType);
+    error_code load(const std::string &dir) WARN_UNUSED_RESULT;
+    error_code store(const std::string &dir);
     std::string to_string();
 
 private:
-    error_code load_json(const std::string &fname, const dsn::utils::FileDataType &fileDataType);
-    error_code store_json(const std::string &fname, const dsn::utils::FileDataType &fileDataType);
+    error_code load_json(const std::string &fname);
+    error_code store_json(const std::string &fname);
 };
 
 class replica_app_info
@@ -88,8 +111,8 @@ class replica_app_info
 
 public:
     replica_app_info(app_info *app) { _app = app; }
-    error_code load(const std::string &fname, const dsn::utils::FileDataType &fileDataType);
-    error_code store(const std::string &fname, const dsn::utils::FileDataType &fileDataType);
+    error_code load(const std::string &fname);
+    error_code store(const std::string &fname);
 };
 
 /// The store engine interface of Pegasus.

diff --git a/src/replica/test/replica_test.cpp b/src/replica/test/replica_test.cpp
@@ -239,7 +239,7 @@ class replica_test : public replica_test_base
         std::cout << "the path of .app-info file is " << path << std::endl;
 
         // load new max_replica_count from file
-        auto err = replica_info.load(path, dsn::utils::FileDataType::kSensitive);
+        auto err = replica_info.load(path);
         ASSERT_EQ(ERR_OK, err);
         ASSERT_EQ(info, _mock_replica->_app_info);
         std::cout << "the loaded new app_info is " << info << std::endl;
@@ -249,7 +249,7 @@ class replica_test : public replica_test_base
         _app_info.max_replica_count = reserved_max_replica_count;
 
         // load original max_replica_count from file
-        err = replica_info.load(path, dsn::utils::FileDataType::kSensitive);
+        err = replica_info.load(path);
         ASSERT_EQ(err, ERR_OK);
         ASSERT_EQ(info, _mock_replica->_app_info);
         std::cout << "the loaded original app_info is " << info << std::endl;