feat: source config_hdfs.sh automatically before starting onebox (#1708)

When run Pegasus to use the feature of backup data to HDFS (restore data from
HDFS is the same), it's needed to set the CLASSPATH enviroment variables, now
it is done by scripts/config_hdfs.sh. However, we have to source this script
manually before starting onebox.

This patch sources the script in ./run.sh automatically, updates config_hdfs.sh
to download hadoop package is it's not exist, and removes explicilt calling of
scripts/config_hdfs.sh in CI.

.github/workflows/lint_and_test_cpp.yaml

@@ -252,7 +252,6 @@ jobs:
         run: |
           export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
           ulimit -s unlimited
-          . ./scripts/config_hdfs.sh
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_ASAN:
@@ -392,7 +391,6 @@ jobs:
         run: |
           export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
           ulimit -s unlimited
-          . ./scripts/config_hdfs.sh
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
 # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
@@ -534,7 +532,6 @@ jobs:
 #        run: |
 #          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
 #          ulimit -s unlimited
-#          . ./scripts/config_hdfs.sh
 #          ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_with_jemalloc:
@@ -647,7 +644,6 @@ jobs:
         run: |
           export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
           ulimit -s unlimited
-          . ./scripts/config_hdfs.sh
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_pegasus_on_macos:

run.sh

@@ -784,7 +784,7 @@ function run_start_onebox()
         exit 1
     fi
 
-    echo "HDFS_SERVICE_ARGS $HDFS_SERVICE_ARGS"
+    source "${ROOT}"/scripts/config_hdfs.sh
     if [ $USE_PRODUCT_CONFIG == "true" ]; then
         [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini
         [ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist"; exit 1; }
@@ -1028,6 +1028,7 @@ function run_start_onebox_instance()
         esac
         shift
     done
+    source "${ROOT}"/scripts/config_hdfs.sh
     if [ $META_ID = "0" -a $REPLICA_ID = "0" -a $COLLECTOR_ID = "0" ]; then
         echo "ERROR: no meta_id or replica_id or collector set"
         exit 1

scripts/config_hdfs.sh

@@ -19,6 +19,8 @@
 
 set -e
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT=$(dirname "${SCRIPT_DIR}")
 # This file should be sourced to set up LD_LIBRARY_PATH and CLASSPATH to
 # run Pegasus binaries which use libhdfs in the context of a dev environment.
 
@@ -39,11 +41,19 @@ fi
 JAVA_JVM_LIBRARY_DIR=$(dirname $(find "${JAVA_HOME}/" -name libjvm.so  | head -1))
 export LD_LIBRARY_PATH=${JAVA_JVM_LIBRARY_DIR}:$LD_LIBRARY_PATH
 
+if [ ! -d "$HADOOP_HOME" ]; then
+  PEGASUS_HADOOP_HOME=${ROOT}/hadoop-bin
+  if [ ! -d "$PEGASUS_HADOOP_HOME" ]; then
+    "${SCRIPT_DIR}"/download_hadoop.sh "${PEGASUS_HADOOP_HOME}"
+  fi
+
+  # Set the HADOOP_HOME to the pegasus's hadoop directory.
+  export HADOOP_HOME="${PEGASUS_HADOOP_HOME}"
+  echo "set HADOOP_HOME to ${PEGASUS_HADOOP_HOME}"
+fi
+
 # Set CLASSPATH to all the Hadoop jars needed to run Hadoop itself as well as
 # the right configuration directory containing core-site.xml or hdfs-site.xml.
-PEGASUS_HADOOP_HOME=`pwd`/hadoop-bin
-# Prefer the HADOOP_HOME set in the environment, but use the pegasus's hadoop dir otherwise.
-export HADOOP_HOME="${HADOOP_HOME:-${PEGASUS_HADOOP_HOME}}"
 if [ ! -d "$HADOOP_HOME/etc/hadoop" ] || [ ! -d "$HADOOP_HOME/share/hadoop" ]; then
   echo "HADOOP_HOME must be set to the location of your Hadoop jars and core-site.xml."
   return 1