feat(slog): Remove slog related code (apache#1759)

Since Pegasus 2.4.0, there will be no new shared logs been created, since Pegasus 2.5.0, all shared logs will garbage collected, then there is no shared logs left. It's a time to remove all shared log related code. This patch also ensure that there is slog left, otherwise, the server will crash when bootstrap. This change should be mentioned as "behavior changes" in release notes.
empiredan · Dec 14, 2023 · 2b7d658 · 2b7d658
1 parent 5eb33cc
commit 2b7d658
Show file tree

Hide file tree

Showing 82 changed files with 165 additions and 1,391 deletions.
diff --git a/docker/pegasus-docker-compose/config.min.ini b/docker/pegasus-docker-compose/config.min.ini
@@ -28,7 +28,7 @@
   type = replica
   name = replica
   ports = 34801
-  pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+  pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_PLOG
 
 [apps.collector]
   name = collector
@@ -105,10 +105,6 @@
   partitioned = false
   worker_count = 2
 
-[threadpool.THREAD_POOL_SLOG]
-  name = slog
-  worker_count = 1
-
 [threadpool.THREAD_POOL_PLOG]
   name = plog
   partitioned = true

diff --git a/src/common/replication.codes.h b/src/common/replication.codes.h
@@ -38,7 +38,6 @@ DEFINE_THREAD_POOL_CODE(THREAD_POOL_LOCAL_APP)
 DEFINE_THREAD_POOL_CODE(THREAD_POOL_REPLICATION_LONG)
 DEFINE_THREAD_POOL_CODE(THREAD_POOL_COMPACT)
 DEFINE_THREAD_POOL_CODE(THREAD_POOL_INGESTION)
-DEFINE_THREAD_POOL_CODE(THREAD_POOL_SLOG)
 DEFINE_THREAD_POOL_CODE(THREAD_POOL_PLOG)
 DEFINE_THREAD_POOL_CODE(THREAD_POOL_SCAN)
 
@@ -207,10 +206,6 @@ MAKE_EVENT_CODE(LPC_REPLICATION_LONG_COMMON, TASK_PRIORITY_COMMON)
 MAKE_EVENT_CODE(LPC_REPLICATION_LONG_HIGH, TASK_PRIORITY_HIGH)
 #undef CURRENT_THREAD_POOL
 
-#define CURRENT_THREAD_POOL THREAD_POOL_SLOG
-MAKE_EVENT_CODE_AIO(LPC_WRITE_REPLICATION_LOG_SHARED, TASK_PRIORITY_HIGH)
-#undef CURRENT_THREAD_POOL
-
 #define CURRENT_THREAD_POOL THREAD_POOL_PLOG
 MAKE_EVENT_CODE_AIO(LPC_WRITE_REPLICATION_LOG_PRIVATE, TASK_PRIORITY_HIGH)
 #undef CURRENT_THREAD_POOL

diff --git a/src/common/replication_common.cpp b/src/common/replication_common.cpp
@@ -26,6 +26,7 @@
 
 #include "common/replication_common.h"
 
+#include <string.h>
 // IWYU pragma: no_include <ext/alloc_traits.h>
 #include <algorithm>
 #include <fstream>
@@ -63,7 +64,7 @@ DSN_DEFINE_int32(
     replication,
     gc_interval_ms,
     30 * 1000,
-    "every what period (ms) we do garbage collection for dead replicas, on-disk state, log, etc.");
+    "every what period (ms) we do replica stat. The name contains 'gc' is for legacy reason.");
 DSN_DEFINE_int32(replication,
                  fd_check_interval_seconds,
                  2,
@@ -99,7 +100,12 @@ DSN_DEFINE_bool(replication,
                 "whether to disable empty write, default is false");
 DSN_TAG_VARIABLE(empty_write_disabled, FT_MUTABLE);
 
-DSN_DEFINE_string(replication, slog_dir, "", "The shared log directory");
+DSN_DEFINE_string(replication,
+                  slog_dir,
+                  "",
+                  "The shared log directory. Deprecated since Pegasus "
+                  "2.6.0, but leave it and do not modify the value if "
+                  "upgrading from older versions.");
 DSN_DEFINE_string(replication, data_dirs, "", "replica directory list");
 DSN_DEFINE_string(replication,
                   data_dirs_black_list_file,
@@ -118,11 +124,7 @@ void replication_options::initialize()
     app_name = info.full_name;
     app_dir = info.data_dir;
 
-    // slog_dir:
-    // - if config[slog_dir] is empty: "app_dir/slog"
-    // - else: "config[slog_dir]/app_name/slog"
-    slog_dir = FLAGS_slog_dir;
-    if (slog_dir.empty()) {
+    if (strlen(FLAGS_slog_dir) == 0) {
         slog_dir = app_dir;
     } else {
         slog_dir = utils::filesystem::path_combine(slog_dir, app_name);

diff --git a/src/failure_detector/test/config-test.ini b/src/failure_detector/test/config-test.ini
@@ -36,15 +36,15 @@ run = true
 ports =
 count = 1
 delay_seconds = 1
-pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_PLOG
 
 [apps.server]
 type = test
 arguments =
 ports = 20101
 run = false
 count = 0
-pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION,THREAD_POOL_PLOG
 
 [apps.test_worker]
 type = worker

diff --git a/src/failure_detector/test/config-whitelist-test-failed.ini b/src/failure_detector/test/config-whitelist-test-failed.ini
@@ -36,15 +36,15 @@ run = true
 ports =
 count = 1
 delay_seconds = 1
-pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_PLOG
 
 [apps.server]
 type = test
 arguments =
 ports = 20101
 run = false
 count = 0
-pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION,THREAD_POOL_PLOG
 
 [apps.test_worker]
 type = worker

diff --git a/src/failure_detector/test/config-whitelist-test.ini b/src/failure_detector/test/config-whitelist-test.ini
@@ -36,15 +36,15 @@ run = true
 ports =
 count = 1
 delay_seconds = 1
-pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_PLOG
 
 [apps.server]
 type = test
 arguments =
 ports = 20101
 run = false
 count = 0
-pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION,THREAD_POOL_PLOG
 
 [apps.test_worker]
 type = worker

diff --git a/src/perf_counter/perf_counters.cpp b/src/perf_counter/perf_counters.cpp
@@ -67,7 +67,6 @@ std::map<std::string, std::string> s_brief_stat_map = {
     {"replica*app.pegasus*manual.compact.running.count", "manual_compact_running_count"},
     {"replica*app.pegasus*manual.compact.enqueue.count", "manual_compact_enqueue_count"},
     {"replica*app.pegasus*rdb.block_cache.memory_usage", "rdb_block_cache_memory_usage"},
-    {"replica*eon.replica_stub*shared.log.size(MB)", "shared_log_size(MB)"},
     {"replica*server*memused.virt(MB)", "memused_virt(MB)"},
     {"replica*server*memused.res(MB)", "memused_res(MB)"},
     {"replica*eon.replica_stub*disk.capacity.total(MB)", "disk_capacity_total(MB)"},

diff --git a/src/replica/backup/test/config-test.ini b/src/replica/backup/test/config-test.ini
@@ -27,7 +27,7 @@ run = true
 
 [apps.replica]
 type = replica
-pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_PLOG
 
 [core]
 tool = nativerun

diff --git a/src/replica/bulk_load/test/config-test.ini b/src/replica/bulk_load/test/config-test.ini
@@ -34,7 +34,7 @@ type = replica
 run = true
 count = 1
 ports = 54321
-pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_PLOG
 
 [core]
 ;tool = simulator

diff --git a/src/replica/duplication/test/config-test.ini b/src/replica/duplication/test/config-test.ini
@@ -27,7 +27,7 @@ run = true
 
 [apps.replica]
 type = replica
-pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_SLOG,THREAD_POOL_PLOG
+pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_PLOG
 
 [core]
 tool = nativerun