diff --git a/.clang-tidy b/.clang-tidy index f6da755646..914dee1198 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -20,7 +20,7 @@ CheckOptions: [] # Disable some checks that are not useful for us now. # They are sorted by names, and should be consistent to build_tools/clang_tidy.py. -Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*,-bugprone-easily-swappable-parameters,-bugprone-lambda-function-name,-bugprone-macro-parentheses,-cert-err58-cpp,-concurrency-mt-unsafe,-cppcoreguidelines-avoid-c-arrays,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-avoid-non-const-global-variables,-cppcoreguidelines-macro-usage,-cppcoreguidelines-non-private-member-variables-in-classes,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-cppcoreguidelines-pro-type-const-cast,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-fuchsia-overloaded-operator,-fuchsia-statically-constructed-objects,-google-readability-avoid-underscore-in-googletest-name,-hicpp-avoid-c-arrays,-hicpp-named-parameter,-hicpp-no-array-decay,-llvm-include-order,-misc-definitions-in-headers,-misc-non-private-member-variables-in-classes,-modernize-avoid-c-arrays,-modernize-replace-disallow-copy-and-assign-macro,-modernize-use-trailing-return-type,-performance-unnecessary-value-param,-readability-function-cognitive-complexity,-readability-identifier-length,-readability-magic-numbers,-readability-named-parameter' +Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*,-bugprone-easily-swappable-parameters,-bugprone-lambda-function-name,-bugprone-macro-parentheses,-cert-err58-cpp,-concurrency-mt-unsafe,-cppcoreguidelines-avoid-c-arrays,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-avoid-non-const-global-variables,-cppcoreguidelines-macro-usage,-cppcoreguidelines-non-private-member-variables-in-classes,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-cppcoreguidelines-pro-type-const-cast,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-fuchsia-overloaded-operator,-fuchsia-statically-constructed-objects,-google-readability-avoid-underscore-in-googletest-name,-hicpp-avoid-c-arrays,-hicpp-named-parameter,-hicpp-no-array-decay,-llvm-include-order,-misc-definitions-in-headers,-misc-non-private-member-variables-in-classes,-misc-unused-parameters,-modernize-avoid-c-arrays,-modernize-replace-disallow-copy-and-assign-macro,-modernize-use-trailing-return-type,-performance-unnecessary-value-param,-readability-function-cognitive-complexity,-readability-identifier-length,-readability-magic-numbers,-readability-named-parameter' ExtraArgs: ExtraArgsBefore: [] FormatStyle: none diff --git a/build_tools/clang_tidy.py b/build_tools/clang_tidy.py index e5ef014b88..09ea434b1d 100755 --- a/build_tools/clang_tidy.py +++ b/build_tools/clang_tidy.py @@ -87,6 +87,7 @@ def tidy_on_path(path): "-llvm-include-order," "-misc-definitions-in-headers," "-misc-non-private-member-variables-in-classes," + "-misc-unused-parameters," "-modernize-avoid-c-arrays," "-modernize-replace-disallow-copy-and-assign-macro," "-modernize-use-trailing-return-type," diff --git a/idl/duplication.thrift b/idl/duplication.thrift index 604edd1d3b..3886021fec 100644 --- a/idl/duplication.thrift +++ b/idl/duplication.thrift @@ -17,6 +17,7 @@ include "dsn.thrift" include "dsn.layer2.thrift" +include "utils.thrift" namespace cpp dsn.replication namespace go admin @@ -179,6 +180,15 @@ struct duplication_entry 11:optional map partition_states; } +// States for the duplications of a table. +struct duplication_app_state +{ + 1:i32 appid; + + // dup id => per-duplication properties + 2:map duplications; +} + // This request is sent from client to meta. struct duplication_query_request { @@ -234,3 +244,21 @@ struct duplication_sync_response // this rpc will not return the apps that were not assigned duplication. 2:map> dup_map; } + +// This request is sent from client to meta server, to list duplications with their +// per-duplication info and progress of each partition for one or multiple tables. +struct duplication_list_request +{ + // The pattern used to match an app name, whose type is specified by `match_type`. + 1:string app_name_pattern; + 2:utils.pattern_match_type match_type; +} + +struct duplication_list_response +{ + 1:dsn.error_code err; + 2:string hint_message; + + // app name => duplications owned by an app + 3:map app_states; +} diff --git a/idl/meta_admin.thrift b/idl/meta_admin.thrift index eec65717e3..6a6af97c7f 100644 --- a/idl/meta_admin.thrift +++ b/idl/meta_admin.thrift @@ -447,6 +447,8 @@ service admin_client duplication.duplication_modify_response modify_duplication(1: duplication.duplication_modify_request req); + duplication.duplication_list_response list_duplication(1: duplication.duplication_list_request req); + query_app_info_response query_app_info(1: query_app_info_request req); configuration_update_app_env_response update_app_env(1: configuration_update_app_env_request req); diff --git a/idl/utils.thrift b/idl/utils.thrift new file mode 100644 index 0000000000..659b2c5141 --- /dev/null +++ b/idl/utils.thrift @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace cpp dsn.utils +namespace go utils +namespace java org.apache.pegasus.utils + +// How a string matches to a given pattern. +enum pattern_match_type +{ + PMT_INVALID = 0, + + // The string always matches no matter what the given pattern is. + PMT_MATCH_ALL, + + // The string must exactly equal to the given pattern. + PMT_MATCH_EXACT, + + // The string must appear anywhere in the given pattern. + PMT_MATCH_ANYWHERE, + + // The string must start with the given pattern. + PMT_MATCH_PREFIX, + + // The string must end with the given pattern. + PMT_MATCH_POSTFIX, + + // The string must match the given pattern as a regular expression. + PMT_MATCH_REGEX, +} diff --git a/src/client/replication_ddl_client.cpp b/src/client/replication_ddl_client.cpp index 3a0fc3a9c5..e993b35684 100644 --- a/src/client/replication_ddl_client.cpp +++ b/src/client/replication_ddl_client.cpp @@ -1416,6 +1416,16 @@ replication_ddl_client::query_dup(const std::string &app_name) return call_rpc_sync(duplication_query_rpc(std::move(req), RPC_CM_QUERY_DUPLICATION)); } +error_with +replication_ddl_client::list_dups(const std::string &app_name_pattern, + utils::pattern_match_type::type match_type) +{ + auto req = std::make_unique(); + req->app_name_pattern = app_name_pattern; + req->match_type = match_type; + return call_rpc_sync(duplication_list_rpc(std::move(req), RPC_CM_LIST_DUPLICATION)); +} + namespace { bool need_retry(uint32_t attempt_count, const dsn::error_code &err) diff --git a/src/client/replication_ddl_client.h b/src/client/replication_ddl_client.h index 2f891da2ec..4aab2ae7c4 100644 --- a/src/client/replication_ddl_client.h +++ b/src/client/replication_ddl_client.h @@ -60,6 +60,7 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/ports.h" +#include "utils_types.h" DSN_DECLARE_uint32(ddl_client_max_attempt_count); DSN_DECLARE_uint32(ddl_client_retry_interval_ms); @@ -154,6 +155,9 @@ class replication_ddl_client error_with query_dup(const std::string &app_name); + error_with list_dups(const std::string &app_name_pattern, + utils::pattern_match_type::type match_type); + dsn::error_code do_restore(const std::string &backup_provider_name, const std::string &cluster_name, const std::string &policy_name, diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 309a12f1b0..836502fd69 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -24,6 +24,12 @@ set(MY_PROJ_NAME dsn_replication_common) +thrift_generate_cpp( + METADATA_THRIFT_SRCS + METADATA_THRIFT_HDRS + ${PROJECT_ROOT}/idl/utils.thrift +) + thrift_generate_cpp( METADATA_THRIFT_SRCS METADATA_THRIFT_HDRS diff --git a/src/common/duplication_common.h b/src/common/duplication_common.h index 0b42daef49..8c13b5a837 100644 --- a/src/common/duplication_common.h +++ b/src/common/duplication_common.h @@ -31,15 +31,15 @@ DSN_DECLARE_uint32(duplicate_log_batch_bytes); -namespace dsn { -namespace replication { +namespace dsn::replication { -typedef rpc_holder duplication_modify_rpc; -typedef rpc_holder duplication_add_rpc; -typedef rpc_holder duplication_query_rpc; -typedef rpc_holder duplication_sync_rpc; +using duplication_modify_rpc = rpc_holder; +using duplication_add_rpc = rpc_holder; +using duplication_query_rpc = rpc_holder; +using duplication_sync_rpc = rpc_holder; +using duplication_list_rpc = rpc_holder; -typedef int32_t dupid_t; +using dupid_t = int32_t; extern const char *duplication_status_to_string(duplication_status::type status); @@ -92,5 +92,4 @@ struct duplication_constants USER_DEFINED_ENUM_FORMATTER(duplication_fail_mode::type) USER_DEFINED_ENUM_FORMATTER(duplication_status::type) -} // namespace replication -} // namespace dsn +} // namespace dsn::replication diff --git a/src/common/replication.codes.h b/src/common/replication.codes.h index 9a13b5c690..a2c29ef458 100644 --- a/src/common/replication.codes.h +++ b/src/common/replication.codes.h @@ -112,6 +112,7 @@ MAKE_EVENT_CODE_RPC(RPC_CM_ADD_DUPLICATION, TASK_PRIORITY_COMMON) MAKE_EVENT_CODE_RPC(RPC_CM_MODIFY_DUPLICATION, TASK_PRIORITY_COMMON) MAKE_EVENT_CODE_RPC(RPC_CM_QUERY_DUPLICATION, TASK_PRIORITY_COMMON) MAKE_EVENT_CODE_RPC(RPC_CM_DUPLICATION_SYNC, TASK_PRIORITY_COMMON) +MAKE_EVENT_CODE_RPC(RPC_CM_LIST_DUPLICATION, TASK_PRIORITY_COMMON) MAKE_EVENT_CODE_RPC(RPC_CM_UPDATE_APP_ENV, TASK_PRIORITY_COMMON) MAKE_EVENT_CODE_RPC(RPC_CM_DDD_DIAGNOSE, TASK_PRIORITY_COMMON) MAKE_EVENT_CODE_RPC(RPC_CM_START_PARTITION_SPLIT, TASK_PRIORITY_COMMON) diff --git a/src/meta/duplication/duplication_info.cpp b/src/meta/duplication/duplication_info.cpp index 5aaa21a9c8..464d9786c3 100644 --- a/src/meta/duplication/duplication_info.cpp +++ b/src/meta/duplication/duplication_info.cpp @@ -296,7 +296,7 @@ void duplication_info::append_as_entry(std::vector &entry_lis { zauto_read_lock l(_lock); - entry_list.emplace_back(to_duplication_level_entry()); + entry_list.emplace_back(to_partition_level_entry_for_list()); } } // namespace dsn::replication diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index 28650dc895..0d01ee2417 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -74,18 +74,69 @@ void meta_duplication_service::query_duplication_info(const duplication_query_re LOG_INFO("query duplication info for app: {}", request.app_name); response.err = ERR_OK; - { - zauto_read_lock l(app_lock()); - std::shared_ptr app = _state->get_app(request.app_name); - if (!app || app->status != app_status::AS_AVAILABLE) { - response.err = ERR_APP_NOT_EXIST; + + zauto_read_lock l(app_lock()); + + std::shared_ptr app = _state->get_app(request.app_name); + if (!app || app->status != app_status::AS_AVAILABLE) { + response.err = ERR_APP_NOT_EXIST; + return; + } + + response.appid = app->app_id; + for (const auto &[_, dup] : app->duplications) { + dup->append_as_entry(response.entry_list); + } +} + +// ThreadPool(READ): THREAD_POOL_META_SERVER +void meta_duplication_service::list_duplication_info(const duplication_list_request &request, + duplication_list_response &response) +{ + LOG_INFO("list duplication info: app_name_pattern={}, match_type={}", + request.app_name_pattern, + enum_to_string(request.match_type)); + + response.err = ERR_OK; + + zauto_read_lock l(app_lock()); + + for (const auto &[app_name, app] : _state->_exist_apps) { + if (app->status != app_status::AS_AVAILABLE) { + // Unavailable tables would not be listed for duplications. + continue; + } + + const auto &err = + utils::pattern_match(app_name, request.app_name_pattern, request.match_type); + if (err == ERR_NOT_MATCHED) { + continue; + } + + if (err == ERR_NOT_IMPLEMENTED) { + const auto &msg = fmt::format("match_type {} is not supported now", + static_cast(request.match_type)); + response.err = err; + response.hint_message = msg; + + LOG_ERROR("{}: app_name_pattern={}", msg, request.app_name_pattern); + + return; + } + + if (err != ERR_OK) { + response.err = err; return; } - response.appid = app->app_id; - for (const auto &[_, dup] : app->duplications) { - dup->append_as_entry(response.entry_list); + duplication_app_state dup_app; + dup_app.appid = app->app_id; + + for (const auto &[dup_id, dup] : app->duplications) { + dup_app.duplications.emplace(dup_id, dup->to_partition_level_entry_for_list()); } + + response.app_states.emplace(app_name, dup_app); } } diff --git a/src/meta/duplication/meta_duplication_service.h b/src/meta/duplication/meta_duplication_service.h index 1cb700d7da..3e70ff8b4b 100644 --- a/src/meta/duplication/meta_duplication_service.h +++ b/src/meta/duplication/meta_duplication_service.h @@ -39,6 +39,8 @@ class zrwlock_nr; namespace replication { class configuration_create_app_response; class duplication_confirm_entry; +class duplication_list_request; +class duplication_list_response; class duplication_query_request; class duplication_query_response; class meta_service; @@ -69,8 +71,12 @@ class meta_duplication_service /// See replication.thrift for possible errors for each rpc. + // Query duplications for one table. void query_duplication_info(const duplication_query_request &, duplication_query_response &); + // List duplications for one or multiple tables. + void list_duplication_info(const duplication_list_request &, duplication_list_response &); + void add_duplication(duplication_add_rpc rpc); void modify_duplication(duplication_modify_rpc rpc); diff --git a/src/meta/meta_service.cpp b/src/meta/meta_service.cpp index 6a07e3c91d..bd9736304b 100644 --- a/src/meta/meta_service.cpp +++ b/src/meta/meta_service.cpp @@ -1038,6 +1038,20 @@ void meta_service::on_duplication_sync(duplication_sync_rpc rpc) server_state::sStateHash); } +void meta_service::on_list_duplication_info(duplication_list_rpc rpc) +{ + if (!check_status_and_authz(rpc)) { + return; + } + + if (!_dup_svc) { + rpc.response().err = ERR_SERVICE_NOT_ACTIVE; + return; + } + + _dup_svc->list_duplication_info(rpc.request(), rpc.response()); +} + void meta_service::recover_duplication_from_meta_state() { if (_dup_svc) { @@ -1056,6 +1070,8 @@ void meta_service::register_duplication_rpc_handlers() &meta_service::on_query_duplication_info); register_rpc_handler_with_rpc_holder( RPC_CM_DUPLICATION_SYNC, "sync duplication", &meta_service::on_duplication_sync); + register_rpc_handler_with_rpc_holder( + RPC_CM_LIST_DUPLICATION, "list_duplication_info", &meta_service::on_list_duplication_info); } void meta_service::initialize_duplication_service() diff --git a/src/meta/meta_service.h b/src/meta/meta_service.h index 309fc06515..5230fbfd4a 100644 --- a/src/meta/meta_service.h +++ b/src/meta/meta_service.h @@ -262,6 +262,7 @@ class meta_service : public serverlet void on_modify_duplication(duplication_modify_rpc rpc); void on_query_duplication_info(duplication_query_rpc rpc); void on_duplication_sync(duplication_sync_rpc rpc); + void on_list_duplication_info(duplication_list_rpc rpc); void register_duplication_rpc_handlers(); void recover_duplication_from_meta_state(); void initialize_duplication_service(); diff --git a/src/meta/test/meta_duplication_service_test.cpp b/src/meta/test/meta_duplication_service_test.cpp index 5e0ebe6b19..7ce7582713 100644 --- a/src/meta/test/meta_duplication_service_test.cpp +++ b/src/meta/test/meta_duplication_service_test.cpp @@ -65,9 +65,9 @@ #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/time_utils.h" +#include "utils_types.h" -namespace dsn { -namespace replication { +namespace dsn::replication { class meta_duplication_service_test : public meta_test_base { @@ -142,6 +142,19 @@ class meta_duplication_service_test : public meta_test_base return rpc.response(); } + duplication_list_response list_dup_info(const std::string &app_name_pattern, + utils::pattern_match_type::type match_type) + { + auto req = std::make_unique(); + req->app_name_pattern = app_name_pattern; + req->match_type = match_type; + + duplication_list_rpc rpc(std::move(req), RPC_CM_LIST_DUPLICATION); + dup_svc().list_duplication_info(rpc.request(), rpc.response()); + + return rpc.response(); + } + duplication_modify_response change_dup_status(const std::string &app_name, dupid_t dupid, duplication_status::type status) { @@ -519,6 +532,48 @@ class meta_duplication_service_test : public meta_test_base ASSERT_TRUE(app->duplicating); } } + + void test_list_dup_app_state(const std::string &app_name, const duplication_app_state &state) + { + const auto &app = find_app(app_name); + + // Each app id should be matched. + ASSERT_EQ(app->app_id, state.appid); + + // The number of returned duplications for each table should be as expected. + ASSERT_EQ(app->duplications.size(), state.duplications.size()); + + for (const auto &[dup_id, dup] : app->duplications) { + // Each dup id should be matched. + ASSERT_TRUE(gutil::ContainsKey(state.duplications, dup_id)); + ASSERT_EQ(dup_id, gutil::FindOrDie(state.duplications, dup_id).dupid); + + // The number of returned partitions should be as expected. + ASSERT_EQ(app->partition_count, + gutil::FindOrDie(state.duplications, dup_id).partition_states.size()); + } + } + + void test_list_dup_info(const std::vector &app_names, + const std::string &app_name_pattern, + utils::pattern_match_type::type match_type) + { + const auto &resp = list_dup_info(app_name_pattern, match_type); + + // Request for listing duplications should be successful. + ASSERT_EQ(ERR_OK, resp.err); + + // The number of returned tables should be as expected. + ASSERT_EQ(app_names.size(), resp.app_states.size()); + + for (const auto &app_name : app_names) { + // Each table name should be in the returned list. + ASSERT_TRUE(gutil::ContainsKey(resp.app_states, app_name)); + + // Test the states of each table. + test_list_dup_app_state(app_name, gutil::FindOrDie(resp.app_states, app_name)); + } + } }; const std::string meta_duplication_service_test::kTestAppName = "test_app"; @@ -796,16 +851,60 @@ TEST_F(meta_duplication_service_test, query_duplication_info) change_dup_status(kTestAppName, test_dup, duplication_status::DS_PAUSE); auto resp = query_dup_info(kTestAppName); - ASSERT_EQ(resp.err, ERR_OK); - ASSERT_EQ(resp.entry_list.size(), 1); - ASSERT_EQ(resp.entry_list.back().status, duplication_status::DS_PREPARE); - ASSERT_EQ(resp.entry_list.back().dupid, test_dup); - ASSERT_EQ(resp.appid, app->app_id); + ASSERT_EQ(ERR_OK, resp.err); + ASSERT_EQ(1, resp.entry_list.size()); + ASSERT_EQ(duplication_status::DS_PREPARE, resp.entry_list.back().status); + ASSERT_EQ(test_dup, resp.entry_list.back().dupid); + ASSERT_EQ(app->app_id, resp.appid); change_dup_status(kTestAppName, test_dup, duplication_status::DS_REMOVED); resp = query_dup_info(kTestAppName); - ASSERT_EQ(resp.err, ERR_OK); - ASSERT_EQ(resp.entry_list.size(), 0); + ASSERT_EQ(ERR_OK, resp.err); + ASSERT_TRUE(resp.entry_list.empty()); +} + +TEST_F(meta_duplication_service_test, list_duplication_info) +{ + // Remove all tables from memory to prevent later tests from interference. + clear_apps(); + + // Create some tables with some partitions and duplications randomly. + create_app(kTestAppName, 8); + create_dup(kTestAppName); + create_dup(kTestAppName); + + std::string app_name_1("test_list_dup"); + create_app(app_name_1, 4); + create_dup(app_name_1); + + std::string app_name_2("list_apps_test"); + create_app(app_name_2, 8); + + std::string app_name_3("test_dup_list"); + create_app(app_name_3, 8); + create_dup(app_name_3); + create_dup(app_name_3); + create_dup(app_name_3); + + // Test for returning all of the existing tables. + test_list_dup_info({kTestAppName, app_name_1, app_name_2, app_name_3}, + {}, + utils::pattern_match_type::PMT_MATCH_ALL); + + // Test for returning the tables whose name are matched exactly. + test_list_dup_info({app_name_2}, app_name_2, utils::pattern_match_type::PMT_MATCH_EXACT); + + // Test for returning the tables whose name are matched with pattern anywhere. + test_list_dup_info( + {kTestAppName, app_name_2}, "app", utils::pattern_match_type::PMT_MATCH_ANYWHERE); + + // Test for returning the tables whose name are matched with pattern as prefix. + test_list_dup_info({kTestAppName, app_name_1, app_name_3}, + "test", + utils::pattern_match_type::PMT_MATCH_PREFIX); + + // Test for returning the tables whose name are matched with pattern as postfix. + test_list_dup_info({app_name_2}, "test", utils::pattern_match_type::PMT_MATCH_POSTFIX); } TEST_F(meta_duplication_service_test, re_add_duplication) @@ -873,9 +972,17 @@ TEST_F(meta_duplication_service_test, query_duplication_handler) static_cast(dup->create_timestamp_ms), ts_buf, sizeof(ts_buf)); ASSERT_EQ(std::string() + R"({"1":{"create_ts":")" + ts_buf + R"(","dupid":)" + std::to_string(dup->id) + - R"(,"fail_mode":"FAIL_SLOW","remote":"slave-cluster")" - R"(,"remote_app_name":"remote_test_app","remote_replica_count":3)" - R"(,"status":"DS_PREPARE"},"appid":2})", + R"(,"fail_mode":"FAIL_SLOW","partition_states":{)" + R"("0":{"confirmed_decree":-1,"last_committed_decree":-1},)" + R"("1":{"confirmed_decree":-1,"last_committed_decree":-1},)" + R"("2":{"confirmed_decree":-1,"last_committed_decree":-1},)" + R"("3":{"confirmed_decree":-1,"last_committed_decree":-1},)" + R"("4":{"confirmed_decree":-1,"last_committed_decree":-1},)" + R"("5":{"confirmed_decree":-1,"last_committed_decree":-1},)" + R"("6":{"confirmed_decree":-1,"last_committed_decree":-1},)" + R"("7":{"confirmed_decree":-1,"last_committed_decree":-1})" + R"(},"remote":"slave-cluster","remote_app_name":"remote_test_app",)" + R"("remote_replica_count":3,"status":"DS_PREPARE"},"appid":2})", fake_resp.body); } @@ -1098,5 +1205,4 @@ TEST_F(meta_duplication_service_test, mark_follower_app_created_for_duplication) } } -} // namespace replication -} // namespace dsn +} // namespace dsn::replication diff --git a/src/meta/test/meta_test_base.cpp b/src/meta/test/meta_test_base.cpp index db136d3913..727c590643 100644 --- a/src/meta/test/meta_test_base.cpp +++ b/src/meta/test/meta_test_base.cpp @@ -222,6 +222,13 @@ void meta_test_base::drop_app(const std::string &name) ASSERT_TRUE(_ss->spin_wait_staging(30)); } +void meta_test_base::clear_apps() +{ + zauto_write_lock l(_ss->_lock); + _ss->_exist_apps.clear(); + _ss->_all_apps.clear(); +} + std::shared_ptr meta_test_base::find_app(const std::string &name) { return _ss->get_app(name); diff --git a/src/meta/test/meta_test_base.h b/src/meta/test/meta_test_base.h index 25f3eed42f..9b62897681 100644 --- a/src/meta/test/meta_test_base.h +++ b/src/meta/test/meta_test_base.h @@ -70,6 +70,8 @@ class meta_test_base : public testing::Test // drop an app for test. void drop_app(const std::string &name); + void clear_apps(); + configuration_update_app_env_response update_app_envs(const std::string &app_name, const std::vector &env_keys, const std::vector &env_vals); diff --git a/src/replica/replica_backup.cpp b/src/replica/replica_backup.cpp index 3e8856bee1..1be16e30c0 100644 --- a/src/replica/replica_backup.cpp +++ b/src/replica/replica_backup.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include diff --git a/src/rpc/rpc_message.cpp b/src/rpc/rpc_message.cpp index 394abf3ed4..4c394f1b90 100644 --- a/src/rpc/rpc_message.cpp +++ b/src/rpc/rpc_message.cpp @@ -41,8 +41,8 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/join_point.h" -#include "utils/strings.h" #include "utils/utils.h" +#include "utils_types.h" // init common for all per-node providers DSN_DEFINE_uint32(core, diff --git a/src/runtime/service_engine.cpp b/src/runtime/service_engine.cpp index 0012951186..71bb160506 100644 --- a/src/runtime/service_engine.cpp +++ b/src/runtime/service_engine.cpp @@ -49,11 +49,9 @@ #include "utils/string_conv.h" #include "utils/strings.h" -using namespace dsn::utils; - namespace dsn { -service_node::service_node(service_app_spec &app_spec) { _app_spec = app_spec; } +service_node::service_node(service_app_spec &app_spec) : _app_spec(app_spec) {} bool service_node::rpc_register_handler(task_code code, const char *extra_name, @@ -211,7 +209,7 @@ void service_engine::init_before_toollets(const service_spec &spec) { _spec = sp void service_engine::init_after_toollets() { // init common providers (second half) - _env = factory_store::create( + _env = utils::factory_store::create( _spec.env_factory_name.c_str(), PROVIDER_TYPE_MAIN, nullptr); tls_dsn.env = _env; } diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 835596cd73..827748efcf 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -2338,6 +2338,7 @@ int64_t pegasus_server_impl::last_flushed_decree() const return static_cast(decree); } +// TODO(wangdan): consider using dsn::utils::pattern_match(). bool pegasus_server_impl::validate_filter(::dsn::apps::filter_type::type filter_type, const ::dsn::blob &filter_pattern, const ::dsn::blob &value) diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index f65c43266c..01c37319d1 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -301,9 +301,9 @@ class pegasus_server_impl : public pegasus_read_service } // return true if the data is valid for the filter - bool validate_filter(::dsn::apps::filter_type::type filter_type, - const ::dsn::blob &filter_pattern, - const ::dsn::blob &value); + static bool validate_filter(::dsn::apps::filter_type::type filter_type, + const ::dsn::blob &filter_pattern, + const ::dsn::blob &value); void update_replica_rocksdb_statistics(); diff --git a/src/server/result_writer.cpp b/src/server/result_writer.cpp index 4d3a9160e0..df78172e9c 100644 --- a/src/server/result_writer.cpp +++ b/src/server/result_writer.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include "pegasus/client.h" diff --git a/src/shell/command_utils.h b/src/shell/command_utils.h index e076c3d32d..2f254a8c9d 100644 --- a/src/shell/command_utils.h +++ b/src/shell/command_utils.h @@ -27,11 +27,13 @@ #include #include "shell/argh.h" +#include "utils/ports.h" #include "utils/strings.h" namespace dsn { class host_port; -} +} // namespace dsn + struct shell_context; inline bool validate_cmd(const argh::parser &cmd, diff --git a/src/utils/error_code.h b/src/utils/error_code.h index dfdc680304..8940f3ace6 100644 --- a/src/utils/error_code.h +++ b/src/utils/error_code.h @@ -187,6 +187,8 @@ DEFINE_ERR_CODE(ERR_DUP_EXIST) DEFINE_ERR_CODE(ERR_HTTP_ERROR) +DEFINE_ERR_CODE(ERR_NOT_MATCHED) + } // namespace dsn USER_DEFINED_STRUCTURE_FORMATTER(::dsn::error_code); diff --git a/src/utils/strings.cpp b/src/utils/strings.cpp index aa73ff520c..a8acd80586 100644 --- a/src/utils/strings.cpp +++ b/src/utils/strings.cpp @@ -25,8 +25,9 @@ */ #include +#include +#include #include -#include #include #include #include @@ -113,6 +114,43 @@ bool mequals(const void *lhs, const void *rhs, size_t n) #undef CHECK_NULL_PTR +error_code pattern_match(const std::string &str, + const std::string &pattern, + pattern_match_type::type match_type) +{ + bool matched = false; + switch (match_type) { + case pattern_match_type::PMT_MATCH_ALL: + // Everything is matched. + matched = true; + break; + + case pattern_match_type::PMT_MATCH_EXACT: + matched = str == pattern; + break; + + case pattern_match_type::PMT_MATCH_ANYWHERE: + matched = boost::algorithm::contains(str, pattern); + break; + + case pattern_match_type::PMT_MATCH_PREFIX: + matched = boost::algorithm::starts_with(str, pattern); + break; + + case pattern_match_type::PMT_MATCH_POSTFIX: + matched = boost::algorithm::ends_with(str, pattern); + break; + + // TODO(wangdan): PMT_MATCH_REGEX would be supported soon. + case pattern_match_type::PMT_MATCH_REGEX: + + default: + return ERR_NOT_IMPLEMENTED; + } + + return matched ? ERR_OK : ERR_NOT_MATCHED; +} + std::string get_last_component(const std::string &input, const char splitters[]) { int index = -1; diff --git a/src/utils/strings.h b/src/utils/strings.h index 85d4a75a9b..49ae0b171c 100644 --- a/src/utils/strings.h +++ b/src/utils/strings.h @@ -34,8 +34,19 @@ #include #include -namespace dsn { -namespace utils { +#include "utils_types.h" +#include "utils/enum_helper.h" +#include "utils/error_code.h" + +namespace dsn::utils { + +ENUM_BEGIN2(pattern_match_type::type, pattern_match_type, pattern_match_type::PMT_INVALID) +ENUM_REG(pattern_match_type::PMT_MATCH_EXACT) +ENUM_REG(pattern_match_type::PMT_MATCH_ANYWHERE) +ENUM_REG(pattern_match_type::PMT_MATCH_PREFIX) +ENUM_REG(pattern_match_type::PMT_MATCH_POSTFIX) +ENUM_REG(pattern_match_type::PMT_MATCH_REGEX) +ENUM_END2(pattern_match_type::type, pattern_match_type) inline bool is_empty(const char *str) { return str == nullptr || *str == '\0'; } @@ -62,6 +73,10 @@ bool iequals(const char *lhs, const std::string &rhs, size_t n); // Decide whether the first n bytes of two memory areas are equal, even if one of them is NULL. bool mequals(const void *lhs, const void *rhs, size_t n); +error_code pattern_match(const std::string &str, + const std::string &pattern, + pattern_match_type::type match_type); + // Split the `input` string by the only character `separator` into tokens. Leading and trailing // spaces of each token will be stripped. Once the token is empty, or become empty after // stripping, an empty string will be added into `output` if `keep_place_holder` is enabled. @@ -125,5 +140,4 @@ std::string find_string_prefix(const std::string &input, char separator); // Decide if there are some space characters in the given string, such as ' ', '\r', '\n' or '\t'. bool has_space(const std::string &str); -} // namespace utils -} // namespace dsn +} // namespace dsn::utils diff --git a/src/utils/test/utils.cpp b/src/utils/test/utils.cpp index cd57087507..c018457e9f 100644 --- a/src/utils/test/utils.cpp +++ b/src/utils/test/utils.cpp @@ -40,13 +40,14 @@ #include "utils/binary_reader.h" #include "utils/binary_writer.h" #include "utils/crc.h" +#include "utils/error_code.h" #include "utils/link.h" #include "utils/rand.h" #include "utils/strings.h" #include "utils/utils.h" +#include "utils_types.h" -namespace dsn { -namespace utils { +namespace dsn::utils { TEST(core, get_last_component) { @@ -247,6 +248,76 @@ INSTANTIATE_TEST_SUITE_P(StringTest, CStringNBytesEqualityTest, testing::ValuesIn(c_string_n_bytes_equality_tests)); +struct pattern_match_case +{ + std::string str; + std::string pattern; + pattern_match_type::type match_type; + error_code expected_err; +}; + +class PatternMatchTest : public testing::TestWithParam +{ +}; + +const std::vector pattern_match_tests = { + // Everything would be matched even if pattern is empty. + {"abc", "", pattern_match_type::PMT_MATCH_ALL, ERR_OK}, + // Everything would be matched even if it is not matched completely. + {"abc", "xyz", pattern_match_type::PMT_MATCH_ALL, ERR_OK}, + // It is matched exactly. + {"abc", "abc", pattern_match_type::PMT_MATCH_EXACT, ERR_OK}, + // Empty string is matched exactly with empty pattern. + {"", "", pattern_match_type::PMT_MATCH_EXACT, ERR_OK}, + // Non-empty string cannot be matched exactly with empty pattern. + {"abc", "", pattern_match_type::PMT_MATCH_EXACT, ERR_NOT_MATCHED}, + // The string whose content is different from pattern would not be matched. + {"abc", "xyz", pattern_match_type::PMT_MATCH_EXACT, ERR_NOT_MATCHED}, + // The pattern as a sub string would not be matched. + {"abc", "ab", pattern_match_type::PMT_MATCH_EXACT, ERR_NOT_MATCHED}, + // It is matched with same prefix for anywhere. + {"abcdef", "ab", pattern_match_type::PMT_MATCH_ANYWHERE, ERR_OK}, + // It is matched with same middle for anywhere. + {"abcdef", "cd", pattern_match_type::PMT_MATCH_ANYWHERE, ERR_OK}, + // It is matched with same postfix for anywhere. + {"abcdef", "ef", pattern_match_type::PMT_MATCH_ANYWHERE, ERR_OK}, + // It is matched with empty content for anywhere. + {"abcdef", "", pattern_match_type::PMT_MATCH_ANYWHERE, ERR_OK}, + // It is not matched with different content for anywhere. + {"abcdef", "xyz", pattern_match_type::PMT_MATCH_ANYWHERE, ERR_NOT_MATCHED}, + // It is matched for prefix. + {"abcdef", "ab", pattern_match_type::PMT_MATCH_PREFIX, ERR_OK}, + // It is not matched with same middle for prefix. + {"abcdef", "cd", pattern_match_type::PMT_MATCH_PREFIX, ERR_NOT_MATCHED}, + // It is not matched with same postfix for prefix. + {"abcdef", "ef", pattern_match_type::PMT_MATCH_PREFIX, ERR_NOT_MATCHED}, + // It is not matched with different content for prefix. + {"abcdef", "xyz", pattern_match_type::PMT_MATCH_PREFIX, ERR_NOT_MATCHED}, + // It is matched with empty content for prefix. + {"abcdef", "", pattern_match_type::PMT_MATCH_PREFIX, ERR_OK}, + // It is matched for postfix. + {"abcdef", "ef", pattern_match_type::PMT_MATCH_POSTFIX, ERR_OK}, + // It is not matched with same prefix for postfix. + {"abcdef", "ab", pattern_match_type::PMT_MATCH_POSTFIX, ERR_NOT_MATCHED}, + // It is not matched with same middle for postfix. + {"abcdef", "cd", pattern_match_type::PMT_MATCH_POSTFIX, ERR_NOT_MATCHED}, + // It is not matched with different content for postfix. + {"abcdef", "xyz", pattern_match_type::PMT_MATCH_PREFIX, ERR_NOT_MATCHED}, + // It is matched with empty content for postfix. + {"abcdef", "", pattern_match_type::PMT_MATCH_POSTFIX, ERR_OK}, + // PMT_MATCH_REGEX is still not supported. + {"unsupported", ".*", pattern_match_type::PMT_MATCH_REGEX, ERR_NOT_IMPLEMENTED}, +}; + +TEST_P(PatternMatchTest, PatternMatch) +{ + const auto &test_case = GetParam(); + const auto actual_err = pattern_match(test_case.str, test_case.pattern, test_case.match_type); + EXPECT_EQ(test_case.expected_err, actual_err); +} + +INSTANTIATE_TEST_SUITE_P(StringTest, PatternMatchTest, testing::ValuesIn(pattern_match_tests)); + // For containers such as std::unordered_set, the expected result will be deduplicated // at initialization. Therefore, it can be used to compare with actual result safely. template @@ -600,5 +671,4 @@ const std::vector has_space_tests = { INSTANTIATE_TEST_SUITE_P(StringTest, HasSpaceTest, testing::ValuesIn(has_space_tests)); -} // namespace utils -} // namespace dsn +} // namespace dsn::utils