From e664b99f2ea72646751609c31ec6e2a6c1e2a0ff Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 00:34:32 +0800 Subject: [PATCH 01/15] dump plog tool --- src/replica/mutation.h | 1 + src/replica/mutation_log.cpp | 4 ++-- src/replica/replica.cpp | 12 ++++++++++++ src/replica/replica.h | 4 ++++ src/replica/replica_base.h | 2 ++ src/replica/replica_context.h | 1 + src/tools/mutation_log_tool.cpp | 7 ++++++- 7 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/replica/mutation.h b/src/replica/mutation.h index d5b7f238ad..190d425e39 100644 --- a/src/replica/mutation.h +++ b/src/replica/mutation.h @@ -207,6 +207,7 @@ class replica; class mutation_queue { public: + explicit mutation_queue() {} mutation_queue(gpid gpid, int max_concurrent_op = 2, bool batch_write_disabled = false); ~mutation_queue() diff --git a/src/replica/mutation_log.cpp b/src/replica/mutation_log.cpp index c65333b465..abcc1a92a4 100644 --- a/src/replica/mutation_log.cpp +++ b/src/replica/mutation_log.cpp @@ -479,14 +479,14 @@ void mutation_log_private::commit_pending_mutations(log_file_ptr &lf, mutation_log::mutation_log(const std::string &dir, int32_t max_log_file_mb, gpid gpid, replica *r) { _dir = dir; - _is_private = (gpid.value() != 0); + _is_private = true; _max_log_file_size_in_bytes = static_cast(max_log_file_mb) * 1024L * 1024L; _min_log_file_size_in_bytes = _max_log_file_size_in_bytes / 10; _owner_replica = r; _private_gpid = gpid; if (r) { - CHECK_EQ(_private_gpid, r->get_gpid()); + // CHECK_EQ(_private_gpid, r->get_gpid()); } mutation_log::init_states(); } diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index 62cd71ae94..5a9c508e47 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -266,6 +266,14 @@ DSN_DECLARE_int32(checkpoint_max_interval_hours); const std::string replica::kAppInfo = ".app-info"; +replica::replica() + : serverlet("replica"), + replica_base(), + _primary_states(), + _potential_secondary_states(this) +{ +} + replica::replica(replica_stub *stub, gpid gpid, const app_info &app, @@ -594,6 +602,10 @@ bool replica::verbose_commit_log() const { return _stub->_verbose_commit_log; } void replica::close() { + if (status() == partition_status::PS_INVALID) { + return; + } + CHECK_PREFIX_MSG(status() == partition_status::PS_ERROR || status() == partition_status::PS_INACTIVE || _disk_migrator->status() == disk_migration_status::IDLE || diff --git a/src/replica/replica.h b/src/replica/replica.h index 2057fca6cf..3982af90b4 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -92,6 +92,7 @@ class learn_notify_response; class learn_request; class learn_response; class learn_state; +class mutation_log_tool; class replica; class replica_backup_manager; class replica_bulk_loader; @@ -309,6 +310,8 @@ class replica : public serverlet, public ref_counter, public replica_ba dir_node *dn, bool need_restore, bool is_duplication_follower = false); + + explicit replica(); error_code initialize_on_new(); error_code initialize_on_load(); error_code init_app_and_prepare_list(bool create_new); @@ -529,6 +532,7 @@ class replica : public serverlet, public ref_counter, public replica_ba private: friend class ::dsn::replication::test::test_checker; + friend class mutation_log_tool; friend class ::dsn::replication::mutation_queue; friend class ::dsn::replication::replica_stub; friend class mock_replica; diff --git a/src/replica/replica_base.h b/src/replica/replica_base.h index 8909197a64..9677307307 100644 --- a/src/replica/replica_base.h +++ b/src/replica/replica_base.h @@ -46,6 +46,8 @@ struct replica_base { } + explicit replica_base() {} + gpid get_gpid() const { return _gpid; } const char *replica_name() const { return _name.c_str(); } diff --git a/src/replica/replica_context.h b/src/replica/replica_context.h index 32da41d38b..88fce667e3 100644 --- a/src/replica/replica_context.h +++ b/src/replica/replica_context.h @@ -89,6 +89,7 @@ typedef std::unordered_map<::dsn::rpc_address, remote_learner_state> learner_map class primary_context { public: + explicit primary_context() {} primary_context(gpid gpid, int max_concurrent_2pc_count = 1, bool batch_write_disabled = false) : next_learning_version(0), write_queue(gpid, max_concurrent_2pc_count, batch_write_disabled), diff --git a/src/tools/mutation_log_tool.cpp b/src/tools/mutation_log_tool.cpp index 3c1177d4d9..c56c9c84ad 100644 --- a/src/tools/mutation_log_tool.cpp +++ b/src/tools/mutation_log_tool.cpp @@ -33,6 +33,7 @@ #include "consensus_types.h" #include "replica/mutation.h" #include "replica/mutation_log.h" +#include "replica/replica.h" #include "runtime/rpc/rpc_message.h" #include "runtime/task/task_spec.h" #include "utils/autoref_ptr.h" @@ -49,7 +50,10 @@ bool mutation_log_tool::dump( std::function callback) { - mutation_log_ptr mlog = new mutation_log_shared(log_dir, 32, false); + app_info ai; + auto rep = new replica(); + auto mlog = std::make_shared( + log_dir, FLAGS_log_private_file_size_mb, gpid(2, 0), rep); error_code err = mlog->open( [mlog, &output, callback](int log_length, mutation_ptr &mu) -> bool { if (mlog->max_decree(mu->data.header.pid) == 0) { @@ -92,6 +96,7 @@ bool mutation_log_tool::dump( }, nullptr); mlog->close(); + delete rep; if (err != dsn::ERR_OK) { output << "ERROR: dump mutation log failed, err = " << err.to_string() << std::endl; return false; From b3fbb15bda23ba0a8f739da83066a5df56966ad5 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 10:48:13 +0800 Subject: [PATCH 02/15] 1 --- src/tools/mutation_log_tool.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tools/mutation_log_tool.cpp b/src/tools/mutation_log_tool.cpp index c56c9c84ad..420a838c36 100644 --- a/src/tools/mutation_log_tool.cpp +++ b/src/tools/mutation_log_tool.cpp @@ -27,10 +27,12 @@ #include "mutation_log_tool.h" #include +#include #include #include "common/gpid.h" #include "consensus_types.h" +#include "dsn.layer2_types.h" #include "replica/mutation.h" #include "replica/mutation_log.h" #include "replica/replica.h" @@ -39,11 +41,14 @@ #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/error_code.h" +#include "utils/flags.h" #include "utils/time_utils.h" namespace dsn { namespace replication { +DSN_DECLARE_int32(log_private_file_size_mb); + bool mutation_log_tool::dump( const std::string &log_dir, std::ostream &output, From 9926aca161f21066b4b917260934fd14e63f3341 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 14:48:15 +0800 Subject: [PATCH 03/15] 1 --- src/replica/mutation_log.cpp | 3 +++ src/replica/replica.cpp | 8 -------- src/replica/replica.h | 1 - src/shell/commands/debugger.cpp | 30 +++++++++++++++++++++++------- src/shell/config.ini | 4 ++-- src/tools/mutation_log_tool.cpp | 20 +++++++++++++++++--- src/tools/mutation_log_tool.h | 3 +++ 7 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/replica/mutation_log.cpp b/src/replica/mutation_log.cpp index abcc1a92a4..4f94d157fd 100644 --- a/src/replica/mutation_log.cpp +++ b/src/replica/mutation_log.cpp @@ -521,6 +521,7 @@ error_code mutation_log::open(replay_callback read_callback, io_failure_callback write_error_callback, const std::map &replay_condition) { + LOG_ERROR("open begin"); CHECK(!_is_opened, "cannot open an opened mutation_log"); CHECK_NULL(_current_log_file, ""); @@ -536,6 +537,7 @@ error_code mutation_log::open(replay_callback read_callback, _log_files.clear(); _io_error_callback = write_error_callback; + LOG_ERROR("_dir: {}", _dir); std::vector file_list; if (!dsn::utils::filesystem::get_subfiles(_dir, file_list, false)) { LOG_ERROR("open mutation_log: get subfiles failed."); @@ -548,6 +550,7 @@ error_code mutation_log::open(replay_callback read_callback, std::sort(file_list.begin(), file_list.end()); + LOG_ERROR("file_list size: {}", file_list.size()); error_code err = ERR_OK; for (auto &fpath : file_list) { log_file_ptr log = log_file::open_read(fpath.c_str(), err); diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index 5a9c508e47..687e7b621a 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -266,14 +266,6 @@ DSN_DECLARE_int32(checkpoint_max_interval_hours); const std::string replica::kAppInfo = ".app-info"; -replica::replica() - : serverlet("replica"), - replica_base(), - _primary_states(), - _potential_secondary_states(this) -{ -} - replica::replica(replica_stub *stub, gpid gpid, const app_info &app, diff --git a/src/replica/replica.h b/src/replica/replica.h index 3982af90b4..cf294a814b 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -311,7 +311,6 @@ class replica : public serverlet, public ref_counter, public replica_ba bool need_restore, bool is_duplication_follower = false); - explicit replica(); error_code initialize_on_new(); error_code initialize_on_load(); error_code init_app_and_prepare_list(bool create_new); diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 4febb46401..46a7114ce9 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -29,6 +29,7 @@ // IWYU pragma: no_include // IWYU pragma: no_include #include +#include #include #include #include @@ -45,6 +46,7 @@ #include #include "base/idl_utils.h" +#include "common/gpid.h" #include "common/replication.codes.h" #include "pegasus_key_schema.h" #include "pegasus_utils.h" @@ -78,7 +80,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) {0, 0, 0, 0}}; bool detailed = false; - std::string input; + std::string slog_dir; std::string output; optind = 0; while (true) { @@ -92,7 +94,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) detailed = true; break; case 'i': - input = optarg; + slog_dir = optarg; break; case 'o': output = optarg; @@ -101,12 +103,25 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) return false; } } - if (input.empty()) { - fprintf(stderr, "ERROR: input is not specified\n"); + if (slog_dir.empty()) { + fprintf(stderr, "ERROR: 'input' is not specified\n"); return false; } - if (!dsn::utils::filesystem::directory_exists(input)) { - fprintf(stderr, "ERROR: input %s is not a directory\n", input.c_str()); + if (!dsn::utils::filesystem::directory_exists(slog_dir)) { + fprintf(stderr, "ERROR: 'input' %s is not a directory\n", slog_dir.c_str()); + return false; + } + + char splitters[] = {'\\', '/', 0}; + auto slog_dir_tmp = slog_dir; + std::string name = dsn::utils::get_last_component(dirname((char *)slog_dir_tmp.c_str()), splitters); + if (name.empty()) { + return false; + } + + char app_type[128]; + int32_t app_id, pidx; + if (3 != sscanf(name.c_str(), "%d.%d.%s", &app_id, &pidx, app_type)) { return false; } @@ -217,8 +232,9 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) }; } + fmt::print(stdout, "slog_dir: {}\n", slog_dir); dsn::replication::mutation_log_tool tool; - bool ret = tool.dump(input, os, callback); + bool ret = tool.dump(slog_dir, dsn::gpid(app_id, pidx), os, callback); if (!ret) { fprintf(stderr, "ERROR: dump failed\n"); } else { diff --git a/src/shell/config.ini b/src/shell/config.ini index 6c6ab5adcd..157d4b09ce 100644 --- a/src/shell/config.ini +++ b/src/shell/config.ini @@ -24,7 +24,7 @@ count = 1 [apps.mimic] type = dsn.app.mimic arguments = -pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER +pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_REPLICATION run = true count = 1 @@ -48,7 +48,7 @@ data_dir = ./pegasus_shell.data short_header = false fast_flush = true max_number_of_log_files_on_disk = 10 -stderr_start_level = LOG_LEVEL_FATAL +stderr_start_level = LOG_LEVEL_INFO [tools.simulator] random_seed = 0 diff --git a/src/tools/mutation_log_tool.cpp b/src/tools/mutation_log_tool.cpp index 420a838c36..143a2c0d17 100644 --- a/src/tools/mutation_log_tool.cpp +++ b/src/tools/mutation_log_tool.cpp @@ -30,17 +30,19 @@ #include #include -#include "common/gpid.h" +#include "common/fs_manager.h" #include "consensus_types.h" #include "dsn.layer2_types.h" #include "replica/mutation.h" #include "replica/mutation_log.h" #include "replica/replica.h" +#include "replica/replica_stub.h" #include "runtime/rpc/rpc_message.h" #include "runtime/task/task_spec.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/error_code.h" +#include "utils/filesystem.h" #include "utils/flags.h" #include "utils/time_utils.h" @@ -51,16 +53,28 @@ DSN_DECLARE_int32(log_private_file_size_mb); bool mutation_log_tool::dump( const std::string &log_dir, + gpid pid, std::ostream &output, std::function callback) { + std::string absolute_path; + if (!utils::filesystem::get_absolute_path(log_dir, absolute_path)) { + output << "ERROR: get absolute path failed" << std::endl; + return false; + } + std::string norm_path; + utils::filesystem::get_normalized_path(absolute_path, norm_path); + auto dn = std::make_shared("", norm_path); app_info ai; - auto rep = new replica(); + ai.__set_app_type("pegasus"); + auto stub = std::make_shared(); + auto *rep = new replica(stub.get(), pid, ai, dn.get(), false, false); auto mlog = std::make_shared( - log_dir, FLAGS_log_private_file_size_mb, gpid(2, 0), rep); + log_dir, FLAGS_log_private_file_size_mb, pid, rep); error_code err = mlog->open( [mlog, &output, callback](int log_length, mutation_ptr &mu) -> bool { + std::cout << "1" << std::endl; if (mlog->max_decree(mu->data.header.pid) == 0) { mlog->set_valid_start_offset_on_open(mu->data.header.pid, 0); } diff --git a/src/tools/mutation_log_tool.h b/src/tools/mutation_log_tool.h index d313bdc4a4..d80a7adb4f 100644 --- a/src/tools/mutation_log_tool.h +++ b/src/tools/mutation_log_tool.h @@ -31,6 +31,8 @@ #include #include +#include "common/gpid.h" + namespace dsn { class message_ex; @@ -41,6 +43,7 @@ class mutation_log_tool public: bool dump(const std::string &log_dir, + gpid pid, std::ostream &output, std::function callback); From d5a70240e5d23f7ddc8a0c78c087228bbb805d47 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 15:00:29 +0800 Subject: [PATCH 04/15] 2 --- src/replica/mutation.h | 1 - src/replica/mutation_log.cpp | 7 ++----- src/replica/replica.h | 3 +-- src/replica/replica_base.h | 2 -- src/replica/replica_context.h | 1 - src/shell/config.ini | 2 +- 6 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/replica/mutation.h b/src/replica/mutation.h index 190d425e39..d5b7f238ad 100644 --- a/src/replica/mutation.h +++ b/src/replica/mutation.h @@ -207,7 +207,6 @@ class replica; class mutation_queue { public: - explicit mutation_queue() {} mutation_queue(gpid gpid, int max_concurrent_op = 2, bool batch_write_disabled = false); ~mutation_queue() diff --git a/src/replica/mutation_log.cpp b/src/replica/mutation_log.cpp index 4f94d157fd..c65333b465 100644 --- a/src/replica/mutation_log.cpp +++ b/src/replica/mutation_log.cpp @@ -479,14 +479,14 @@ void mutation_log_private::commit_pending_mutations(log_file_ptr &lf, mutation_log::mutation_log(const std::string &dir, int32_t max_log_file_mb, gpid gpid, replica *r) { _dir = dir; - _is_private = true; + _is_private = (gpid.value() != 0); _max_log_file_size_in_bytes = static_cast(max_log_file_mb) * 1024L * 1024L; _min_log_file_size_in_bytes = _max_log_file_size_in_bytes / 10; _owner_replica = r; _private_gpid = gpid; if (r) { - // CHECK_EQ(_private_gpid, r->get_gpid()); + CHECK_EQ(_private_gpid, r->get_gpid()); } mutation_log::init_states(); } @@ -521,7 +521,6 @@ error_code mutation_log::open(replay_callback read_callback, io_failure_callback write_error_callback, const std::map &replay_condition) { - LOG_ERROR("open begin"); CHECK(!_is_opened, "cannot open an opened mutation_log"); CHECK_NULL(_current_log_file, ""); @@ -537,7 +536,6 @@ error_code mutation_log::open(replay_callback read_callback, _log_files.clear(); _io_error_callback = write_error_callback; - LOG_ERROR("_dir: {}", _dir); std::vector file_list; if (!dsn::utils::filesystem::get_subfiles(_dir, file_list, false)) { LOG_ERROR("open mutation_log: get subfiles failed."); @@ -550,7 +548,6 @@ error_code mutation_log::open(replay_callback read_callback, std::sort(file_list.begin(), file_list.end()); - LOG_ERROR("file_list size: {}", file_list.size()); error_code err = ERR_OK; for (auto &fpath : file_list) { log_file_ptr log = log_file::open_read(fpath.c_str(), err); diff --git a/src/replica/replica.h b/src/replica/replica.h index cf294a814b..c4f3d2ff49 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -310,7 +310,6 @@ class replica : public serverlet, public ref_counter, public replica_ba dir_node *dn, bool need_restore, bool is_duplication_follower = false); - error_code initialize_on_new(); error_code initialize_on_load(); error_code init_app_and_prepare_list(bool create_new); @@ -531,7 +530,7 @@ class replica : public serverlet, public ref_counter, public replica_ba private: friend class ::dsn::replication::test::test_checker; - friend class mutation_log_tool; + friend class ::dsn::replication::mutation_log_tool; friend class ::dsn::replication::mutation_queue; friend class ::dsn::replication::replica_stub; friend class mock_replica; diff --git a/src/replica/replica_base.h b/src/replica/replica_base.h index 9677307307..8909197a64 100644 --- a/src/replica/replica_base.h +++ b/src/replica/replica_base.h @@ -46,8 +46,6 @@ struct replica_base { } - explicit replica_base() {} - gpid get_gpid() const { return _gpid; } const char *replica_name() const { return _name.c_str(); } diff --git a/src/replica/replica_context.h b/src/replica/replica_context.h index 88fce667e3..32da41d38b 100644 --- a/src/replica/replica_context.h +++ b/src/replica/replica_context.h @@ -89,7 +89,6 @@ typedef std::unordered_map<::dsn::rpc_address, remote_learner_state> learner_map class primary_context { public: - explicit primary_context() {} primary_context(gpid gpid, int max_concurrent_2pc_count = 1, bool batch_write_disabled = false) : next_learning_version(0), write_queue(gpid, max_concurrent_2pc_count, batch_write_disabled), diff --git a/src/shell/config.ini b/src/shell/config.ini index 157d4b09ce..8126688393 100644 --- a/src/shell/config.ini +++ b/src/shell/config.ini @@ -48,7 +48,7 @@ data_dir = ./pegasus_shell.data short_header = false fast_flush = true max_number_of_log_files_on_disk = 10 -stderr_start_level = LOG_LEVEL_INFO +stderr_start_level = LOG_LEVEL_FATAL [tools.simulator] random_seed = 0 From 597fa56004ac04f16d1f50b6b85fe12746d824c7 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 15:01:28 +0800 Subject: [PATCH 05/15] 3 --- src/shell/commands/debugger.cpp | 3 ++- src/tools/mutation_log_tool.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 46a7114ce9..951a2eb726 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -114,7 +114,8 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) char splitters[] = {'\\', '/', 0}; auto slog_dir_tmp = slog_dir; - std::string name = dsn::utils::get_last_component(dirname((char *)slog_dir_tmp.c_str()), splitters); + std::string name = + dsn::utils::get_last_component(dirname((char *)slog_dir_tmp.c_str()), splitters); if (name.empty()) { return false; } diff --git a/src/tools/mutation_log_tool.cpp b/src/tools/mutation_log_tool.cpp index 143a2c0d17..74f2a5bea3 100644 --- a/src/tools/mutation_log_tool.cpp +++ b/src/tools/mutation_log_tool.cpp @@ -70,8 +70,8 @@ bool mutation_log_tool::dump( ai.__set_app_type("pegasus"); auto stub = std::make_shared(); auto *rep = new replica(stub.get(), pid, ai, dn.get(), false, false); - auto mlog = std::make_shared( - log_dir, FLAGS_log_private_file_size_mb, pid, rep); + auto mlog = + std::make_shared(log_dir, FLAGS_log_private_file_size_mb, pid, rep); error_code err = mlog->open( [mlog, &output, callback](int log_length, mutation_ptr &mu) -> bool { std::cout << "1" << std::endl; From aa2b54a775e54ace370d0b34382627d1654e97ab Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 15:24:43 +0800 Subject: [PATCH 06/15] fmt --- src/replica/replica.cpp | 4 ---- src/shell/commands/debugger.cpp | 33 +++++++++++++++++---------------- src/shell/main.cpp | 2 +- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index 687e7b621a..62cd71ae94 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -594,10 +594,6 @@ bool replica::verbose_commit_log() const { return _stub->_verbose_commit_log; } void replica::close() { - if (status() == partition_status::PS_INVALID) { - return; - } - CHECK_PREFIX_MSG(status() == partition_status::PS_ERROR || status() == partition_status::PS_INACTIVE || _disk_migrator->status() == disk_migration_status::IDLE || diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 951a2eb726..bbab395900 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -104,11 +104,11 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) } } if (slog_dir.empty()) { - fprintf(stderr, "ERROR: 'input' is not specified\n"); + fmt::print(stderr, "ERROR: 'input' is not specified\n"); return false; } if (!dsn::utils::filesystem::directory_exists(slog_dir)) { - fprintf(stderr, "ERROR: 'input' %s is not a directory\n", slog_dir.c_str()); + fmt::print(stderr, "ERROR: '{}' is not a directory\n", slog_dir); return false; } @@ -117,12 +117,14 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) std::string name = dsn::utils::get_last_component(dirname((char *)slog_dir_tmp.c_str()), splitters); if (name.empty()) { + fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", slog_dir); return false; } char app_type[128]; int32_t app_id, pidx; if (3 != sscanf(name.c_str(), "%d.%d.%s", &app_id, &pidx, app_type)) { + fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", slog_dir); return false; } @@ -132,7 +134,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) } else { os_ptr = new std::ofstream(output); if (!*os_ptr) { - fprintf(stderr, "ERROR: open output file %s failed\n", output.c_str()); + fmt::print(stderr, "ERROR: open output file {} failed\n", output); delete os_ptr; return true; } @@ -233,13 +235,12 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) }; } - fmt::print(stdout, "slog_dir: {}\n", slog_dir); dsn::replication::mutation_log_tool tool; bool ret = tool.dump(slog_dir, dsn::gpid(app_id, pidx), os, callback); if (!ret) { - fprintf(stderr, "ERROR: dump failed\n"); + fmt::print(stderr, "ERROR: dump failed\n"); } else { - fprintf(stderr, "Done\n"); + fmt::print(stderr, "Done\n"); } if (os_ptr != &std::cout) { @@ -263,7 +264,7 @@ bool local_get(command_executor *e, shell_context *sc, arguments args) rocksdb::DB *db; rocksdb::Status status = rocksdb::DB::OpenForReadOnly(db_opts, db_path, &db); if (!status.ok()) { - fprintf(stderr, "ERROR: open db failed: %s\n", status.ToString().c_str()); + fmt::print(stderr, "ERROR: open db failed: {}\n", status.ToString()); return true; } @@ -274,15 +275,15 @@ bool local_get(command_executor *e, shell_context *sc, arguments args) rocksdb::ReadOptions rd_opts; status = db->Get(rd_opts, skey, &value); if (!status.ok()) { - fprintf(stderr, "ERROR: get failed: %s\n", status.ToString().c_str()); + fmt::print(stderr, "ERROR: get failed: {}\n", status.ToString()); } else { uint32_t expire_ts = pegasus::pegasus_extract_expire_ts(0, value); dsn::blob user_data; pegasus::pegasus_extract_user_data(0, std::move(value), user_data); - fprintf(stderr, - "%u : \"%s\"\n", + fmt::print(stderr, + "{} : \"{}\"\n", expire_ts, - pegasus::utils::c_escape_string(user_data, sc->escape_all).c_str()); + pegasus::utils::c_escape_string(user_data, sc->escape_all)); } delete db; @@ -299,7 +300,7 @@ bool rdb_key_str2hex(command_executor *e, shell_context *sc, arguments args) ::dsn::blob key; pegasus::pegasus_generate_key(key, hash_key, sort_key); rocksdb::Slice skey(key.data(), key.length()); - fprintf(stderr, "\"%s\"\n", skey.ToString(true).c_str()); + fmt::print(stderr, "\"{}\"\n", skey.ToString(true)); return true; } @@ -329,12 +330,12 @@ bool rdb_value_hex2str(command_executor *e, shell_context *sc, arguments args) auto expire_ts = static_cast(pegasus::pegasus_extract_expire_ts(0, pegasus_value)) + pegasus::utils::epoch_begin; // TODO(wutao): pass user specified version std::time_t tm(expire_ts); - fmt::print(stderr, "\nWhen to expire:\n {:%Y-%m-%d %H:%M:%S}\n", *std::localtime(&tm)); + fmt::print(stderr, "\nWhen to expire:\n {:%Y-%m-%d %H:%M:%S}\n", fmt::localtime(tm)); dsn::blob user_data; pegasus::pegasus_extract_user_data(0, std::move(pegasus_value), user_data); - fprintf(stderr, - "user_data:\n \"%s\"\n", - pegasus::utils::c_escape_string(user_data.to_string(), sc->escape_all).c_str()); + fmt::print(stderr, + "user_data:\n \"{}\"\n", + pegasus::utils::c_escape_string(user_data.to_string(), sc->escape_all)); return true; } diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 50c1614c10..e7836a5f9c 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -392,7 +392,7 @@ static command_executor commands[] = { { "mlog_dump", "dump mutation log dir", - "<-i|--input log_dir> [-o|--output file_name] [-d|--detailed]", + "<-i|--input log_dir(e.g. '/path/to/replica/reps/2.1.pegasus/plog/')> [-o|--output file_name] [-d|--detailed]", mlog_dump, }, { From 1e6253205b04de60385984b43ad39eeb1ea271a7 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 15:41:34 +0800 Subject: [PATCH 07/15] 4 --- src/shell/commands/debugger.cpp | 10 +++---- src/shell/main.cpp | 3 ++- src/tools/mutation_log_tool.cpp | 46 +++++++++++++++++++-------------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index bbab395900..45f3f522e4 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -281,9 +281,9 @@ bool local_get(command_executor *e, shell_context *sc, arguments args) dsn::blob user_data; pegasus::pegasus_extract_user_data(0, std::move(value), user_data); fmt::print(stderr, - "{} : \"{}\"\n", - expire_ts, - pegasus::utils::c_escape_string(user_data, sc->escape_all)); + "{} : \"{}\"\n", + expire_ts, + pegasus::utils::c_escape_string(user_data, sc->escape_all)); } delete db; @@ -335,7 +335,7 @@ bool rdb_value_hex2str(command_executor *e, shell_context *sc, arguments args) dsn::blob user_data; pegasus::pegasus_extract_user_data(0, std::move(pegasus_value), user_data); fmt::print(stderr, - "user_data:\n \"{}\"\n", - pegasus::utils::c_escape_string(user_data.to_string(), sc->escape_all)); + "user_data:\n \"{}\"\n", + pegasus::utils::c_escape_string(user_data.to_string(), sc->escape_all)); return true; } diff --git a/src/shell/main.cpp b/src/shell/main.cpp index e7836a5f9c..75a52abc63 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -392,7 +392,8 @@ static command_executor commands[] = { { "mlog_dump", "dump mutation log dir", - "<-i|--input log_dir(e.g. '/path/to/replica/reps/2.1.pegasus/plog/')> [-o|--output file_name] [-d|--detailed]", + "<-i|--input log_dir(e.g. '/path/to/replica/reps/2.1.pegasus/plog/')> [-o|--output " + "file_name] [-d|--detailed]", mlog_dump, }, { diff --git a/src/tools/mutation_log_tool.cpp b/src/tools/mutation_log_tool.cpp index 74f2a5bea3..72ac90d895 100644 --- a/src/tools/mutation_log_tool.cpp +++ b/src/tools/mutation_log_tool.cpp @@ -41,6 +41,7 @@ #include "runtime/task/task_spec.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" +#include "utils/defer.h" #include "utils/error_code.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -60,38 +61,46 @@ bool mutation_log_tool::dump( { std::string absolute_path; if (!utils::filesystem::get_absolute_path(log_dir, absolute_path)) { - output << "ERROR: get absolute path failed" << std::endl; + output << fmt::format("ERROR: get absolute path failed\n"); return false; } std::string norm_path; utils::filesystem::get_normalized_path(absolute_path, norm_path); - auto dn = std::make_shared("", norm_path); + auto dn = std::make_unique(/* tag_ */ "", norm_path); app_info ai; ai.__set_app_type("pegasus"); - auto stub = std::make_shared(); - auto *rep = new replica(stub.get(), pid, ai, dn.get(), false, false); + auto stub = std::make_unique(); + // Constructor of replica is private which can not be accessed by std::make_unique, so use raw + // pointer here. + auto *rep = new replica(stub.get(), + pid, + ai, + dn.get(), + /* need_restore */ false, + /* is_duplication_follower */ false); + auto cleanup = dsn::defer([rep]() { delete rep; }); auto mlog = std::make_shared(log_dir, FLAGS_log_private_file_size_mb, pid, rep); error_code err = mlog->open( [mlog, &output, callback](int log_length, mutation_ptr &mu) -> bool { - std::cout << "1" << std::endl; if (mlog->max_decree(mu->data.header.pid) == 0) { mlog->set_valid_start_offset_on_open(mu->data.header.pid, 0); } char timestamp_buf[32] = {0}; utils::time_ms_to_string(mu->data.header.timestamp / 1000, timestamp_buf); - output << "mutation [" << mu->name() << "]: " - << "gpid=" << mu->data.header.pid.get_app_id() << "." - << mu->data.header.pid.get_partition_index() << ", " - << "ballot=" << mu->data.header.ballot << ", decree=" << mu->data.header.decree - << ", " - << "timestamp=" << timestamp_buf - << ", last_committed_decree=" << mu->data.header.last_committed_decree << ", " - << "log_offset=" << mu->data.header.log_offset << ", log_length=" << log_length - << ", " - << "update_count=" << mu->data.updates.size(); - if (callback && mu->data.updates.size() > 0) { - + output << fmt::format("mutation [{}]: gpid={}, ballot={}, decree={}, timestamp={}, " + "last_committed_decree={}, log_offset={}, log_length={}, " + "update_count={}\n", + mu->name(), + mu->data.header.pid, + mu->data.header.ballot, + mu->data.header.decree, + timestamp_buf, + mu->data.header.last_committed_decree, + mu->data.header.log_offset, + log_length, + mu->data.updates.size()); + if (callback && !mu->data.updates.empty()) { dsn::message_ex **batched_requests = (dsn::message_ex **)alloca(sizeof(dsn::message_ex *) * mu->data.updates.size()); int batched_count = 0; @@ -115,9 +124,8 @@ bool mutation_log_tool::dump( }, nullptr); mlog->close(); - delete rep; if (err != dsn::ERR_OK) { - output << "ERROR: dump mutation log failed, err = " << err.to_string() << std::endl; + output << fmt::format("ERROR: dump mutation log failed, err = {}\n", err); return false; } else { return true; From e4195795cad5beff12a49b80cbf5fd133ae80a51 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 15:50:13 +0800 Subject: [PATCH 08/15] iwyu --- src/shell/commands/debugger.cpp | 1 + src/tools/mutation_log_tool.cpp | 2 ++ src/tools/mutation_log_tool.h | 1 + 3 files changed, 4 insertions(+) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 45f3f522e4..e5a6413253 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -64,6 +64,7 @@ #include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" +#include "utils/strings.h" bool sst_dump(command_executor *e, shell_context *sc, arguments args) { diff --git a/src/tools/mutation_log_tool.cpp b/src/tools/mutation_log_tool.cpp index 72ac90d895..b68a4979b5 100644 --- a/src/tools/mutation_log_tool.cpp +++ b/src/tools/mutation_log_tool.cpp @@ -31,8 +31,10 @@ #include #include "common/fs_manager.h" +#include "common/gpid.h" #include "consensus_types.h" #include "dsn.layer2_types.h" +#include "fmt/core.h" #include "replica/mutation.h" #include "replica/mutation_log.h" #include "replica/replica.h" diff --git a/src/tools/mutation_log_tool.h b/src/tools/mutation_log_tool.h index d80a7adb4f..5ac17ac2bc 100644 --- a/src/tools/mutation_log_tool.h +++ b/src/tools/mutation_log_tool.h @@ -34,6 +34,7 @@ #include "common/gpid.h" namespace dsn { +class gpid; class message_ex; namespace replication { From f6042671b123d8f896cef438600f2db5670715c8 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 8 Dec 2023 17:34:34 +0800 Subject: [PATCH 09/15] iwyu --- src/tools/mutation_log_tool.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tools/mutation_log_tool.h b/src/tools/mutation_log_tool.h index 5ac17ac2bc..dee45b456f 100644 --- a/src/tools/mutation_log_tool.h +++ b/src/tools/mutation_log_tool.h @@ -31,7 +31,7 @@ #include #include -#include "common/gpid.h" +#include "common/gpid.h" // IWYU pragma: keep namespace dsn { class gpid; @@ -49,5 +49,5 @@ class mutation_log_tool std::function callback); }; -} -} +} // namespace replication +} // namespace dsn From 2e2f835e9650a2cdc7fc4d12e4c53dbc6c860bf6 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 11 Dec 2023 10:46:57 +0800 Subject: [PATCH 10/15] iwyu --- src/tools/mutation_log_tool.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tools/mutation_log_tool.h b/src/tools/mutation_log_tool.h index dee45b456f..e380be2afb 100644 --- a/src/tools/mutation_log_tool.h +++ b/src/tools/mutation_log_tool.h @@ -31,8 +31,6 @@ #include #include -#include "common/gpid.h" // IWYU pragma: keep - namespace dsn { class gpid; class message_ex; From 2985129bdf5a04d86445342001fee85aa7cfc045 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 12 Dec 2023 12:25:14 +0800 Subject: [PATCH 11/15] rename --- src/shell/commands/debugger.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index e5a6413253..c486f88503 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -81,7 +81,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) {0, 0, 0, 0}}; bool detailed = false; - std::string slog_dir; + std::string plog_dir; std::string output; optind = 0; while (true) { @@ -95,7 +95,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) detailed = true; break; case 'i': - slog_dir = optarg; + plog_dir = optarg; break; case 'o': output = optarg; @@ -104,28 +104,28 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) return false; } } - if (slog_dir.empty()) { + if (plog_dir.empty()) { fmt::print(stderr, "ERROR: 'input' is not specified\n"); return false; } - if (!dsn::utils::filesystem::directory_exists(slog_dir)) { - fmt::print(stderr, "ERROR: '{}' is not a directory\n", slog_dir); + if (!dsn::utils::filesystem::directory_exists(plog_dir)) { + fmt::print(stderr, "ERROR: '{}' is not a directory\n", plog_dir); return false; } char splitters[] = {'\\', '/', 0}; - auto slog_dir_tmp = slog_dir; + auto slog_dir_tmp = plog_dir; std::string name = dsn::utils::get_last_component(dirname((char *)slog_dir_tmp.c_str()), splitters); if (name.empty()) { - fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", slog_dir); + fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", plog_dir); return false; } char app_type[128]; int32_t app_id, pidx; if (3 != sscanf(name.c_str(), "%d.%d.%s", &app_id, &pidx, app_type)) { - fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", slog_dir); + fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", plog_dir); return false; } @@ -237,7 +237,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) } dsn::replication::mutation_log_tool tool; - bool ret = tool.dump(slog_dir, dsn::gpid(app_id, pidx), os, callback); + bool ret = tool.dump(plog_dir, dsn::gpid(app_id, pidx), os, callback); if (!ret) { fmt::print(stderr, "ERROR: dump failed\n"); } else { From f1a765d6e1d34878986ee8ff736672d550748b2a Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 12 Dec 2023 14:53:03 +0800 Subject: [PATCH 12/15] Update src/shell/commands/debugger.cpp Co-authored-by: Dan Wang --- src/shell/commands/debugger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index c486f88503..8575ca6ced 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -118,7 +118,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) std::string name = dsn::utils::get_last_component(dirname((char *)slog_dir_tmp.c_str()), splitters); if (name.empty()) { - fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", plog_dir); + fmt::print(stderr, "ERROR: '{}' is not a valid plog directory\n", plog_dir); return false; } From 270b7067dac09e54cd541c87c0b54abd90e37952 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 12 Dec 2023 14:53:09 +0800 Subject: [PATCH 13/15] Update src/shell/commands/debugger.cpp Co-authored-by: Dan Wang --- src/shell/commands/debugger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 8575ca6ced..ea419e8641 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -125,7 +125,7 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) char app_type[128]; int32_t app_id, pidx; if (3 != sscanf(name.c_str(), "%d.%d.%s", &app_id, &pidx, app_type)) { - fmt::print(stderr, "ERROR: '{}' is not a valid slog directory\n", plog_dir); + fmt::print(stderr, "ERROR: '{}' is not a valid plog directory\n", plog_dir); return false; } From 29e681cbe64a2d3c9c9919c9cff8358d99b5ed16 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 12 Dec 2023 14:53:40 +0800 Subject: [PATCH 14/15] Update src/shell/commands/debugger.cpp Co-authored-by: Dan Wang --- src/shell/commands/debugger.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index ea419e8641..86623b38dd 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -113,10 +113,8 @@ bool mlog_dump(command_executor *e, shell_context *sc, arguments args) return false; } - char splitters[] = {'\\', '/', 0}; - auto slog_dir_tmp = plog_dir; - std::string name = - dsn::utils::get_last_component(dirname((char *)slog_dir_tmp.c_str()), splitters); + const auto replica_path = std::filesystem::path(plog_dir).parent_path(); + const auto name = replica_path.filename().string(); if (name.empty()) { fmt::print(stderr, "ERROR: '{}' is not a valid plog directory\n", plog_dir); return false; From d93b13de162e6705e06ea011fd7882cdbb2e81a9 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 12 Dec 2023 16:52:08 +0800 Subject: [PATCH 15/15] 1 --- src/shell/commands/debugger.cpp | 10 +++------- src/utils/time_utils.cpp | 3 --- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 86623b38dd..8bfe9b2958 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -20,16 +20,10 @@ // IWYU pragma: no_include // TODO(yingchun): refactor this after libfmt upgraded #include // IWYU pragma: keep -// IWYU pragma: no_include -// IWYU pragma: no_include -#if FMT_VERSION < 60000 -#include // IWYU pragma: keep -#endif #include // IWYU pragma: keep // IWYU pragma: no_include // IWYU pragma: no_include #include -#include #include #include #include @@ -39,6 +33,9 @@ #include #include #include +// IWYU pragma: no_include +// IWYU pragma: no_include +#include #include #include #include @@ -64,7 +61,6 @@ #include "utils/blob.h" #include "utils/filesystem.h" #include "utils/fmt_logging.h" -#include "utils/strings.h" bool sst_dump(command_executor *e, shell_context *sc, arguments args) { diff --git a/src/utils/time_utils.cpp b/src/utils/time_utils.cpp index 711b4cd3d8..34504fc993 100644 --- a/src/utils/time_utils.cpp +++ b/src/utils/time_utils.cpp @@ -23,9 +23,6 @@ #include // IWYU pragma: keep // IWYU pragma: no_include // IWYU pragma: no_include -#if FMT_VERSION < 60000 -#include // IWYU pragma: keep -#endif #include // IWYU pragma: keep // IWYU pragma: no_include // IWYU pragma: no_include