From 19876094e941b23894422ae3fdd4b102a9a0bc44 Mon Sep 17 00:00:00 2001
From: Yingchun Lai <laiyingchun@apache.org>
Date: Thu, 30 Nov 2023 17:44:11 +0800
Subject: [PATCH] feat(HDFS): Add hdfs_service related configs in server.ini
 and run.sh (#1705)

This patch adds `hdfs_service` related configs to `src/server/config.ini`, and
add an option `--hdfs_service_args` to `start_onebox` command of `run.sh`.
---
 run.sh                | 10 ++++++++++
 src/server/config.ini | 10 +++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/run.sh b/run.sh
index 74d44b4b42..3c86f669ca 100755
--- a/run.sh
+++ b/run.sh
@@ -690,6 +690,8 @@ function usage_start_onebox()
     echo "                                                                  ./src/server/config.ini in production env"
     echo "   --use_product_config"
     echo "                     use the product config template"
+    echo "   --hdfs_service_args"
+    echo "                     set the 'args' value of section '[block_service.hdfs_service]', it's a space separated HDFS namenode host:port and path string, for example: '127.0.0.1:8020 /pegasus'. Default is empty"
     echo "   --opts"
     echo "                     update configs before start onebox, the configs are in the form of 'key1=value1,key2=value2'"
 }
@@ -705,6 +707,7 @@ function run_start_onebox()
     SERVER_PATH=${BUILD_LATEST_DIR}/output/bin/pegasus_server
     CONFIG_FILE=""
     USE_PRODUCT_CONFIG=false
+    HDFS_SERVICE_ARGS=""
     OPTS=""
 
     while [[ $# > 0 ]]; do
@@ -747,6 +750,11 @@ function run_start_onebox()
             --use_product_config)
                 USE_PRODUCT_CONFIG=true
                 ;;
+            --hdfs_service_args)
+                HDFS_SERVICE_ARGS="$2 $3"
+                shift
+                shift
+                ;;
             --opts)
                 OPTS="$2"
                 shift
@@ -776,6 +784,7 @@ function run_start_onebox()
         exit 1
     fi
 
+    echo "HDFS_SERVICE_ARGS $HDFS_SERVICE_ARGS"
     if [ $USE_PRODUCT_CONFIG == "true" ]; then
         [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini
         [ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist"; exit 1; }
@@ -789,6 +798,7 @@ function run_start_onebox()
         sed -i 's/%{slog.dir}//g' ${ROOT}/config-server.ini
         sed -i 's/%{data.dirs}//g' ${ROOT}/config-server.ini
         sed -i 's@%{home.dir}@'"$HOME"'@g' ${ROOT}/config-server.ini
+        sed -i 's@%{hdfs_service_args}@'"${HDFS_SERVICE_ARGS}"'@g' ${ROOT}/config-server.ini
         for i in $(seq ${META_COUNT})
         do
             meta_port=$((34600+i))
diff --git a/src/server/config.ini b/src/server/config.ini
index 99ca1ea1c7..b9eabf71bb 100644
--- a/src/server/config.ini
+++ b/src/server/config.ini
@@ -301,9 +301,17 @@ stateful = true
   ;; recommand using cluster name as the root
   cold_backup_root = %{cluster.name}
   max_concurrent_uploading_file_count = 10
-
   max_concurrent_bulk_load_downloading_count = 5
 
+  hdfs_read_limit_rate_mb_per_sec = 200
+  hdfs_read_batch_size_bytes = 67108864
+  hdfs_write_limit_rate_mb_per_sec = 200
+  hdfs_write_batch_size_bytes = 67108864
+
+[block_service.hdfs_service]
+  type = hdfs_service
+  args = %{hdfs_service_args}
+
 [pegasus.server]
   rocksdb_verbose_log = false