diff --git a/base-ubuntu-2204/Dockerfile b/base-ubuntu-2204/Dockerfile index 29f6d42..f7fe664 100644 --- a/base-ubuntu-2204/Dockerfile +++ b/base-ubuntu-2204/Dockerfile @@ -62,4 +62,6 @@ RUN groupadd hadoop --gid 6000 && \ groupadd flink --gid 6012 && \ useradd flink --gid 6012 --uid 6012 -m && \ groupadd trino --gid 6013 && \ - useradd trino --gid 6013 --uid 6013 -m + useradd trino --gid 6013 --uid 6013 -m && \ + groupadd hbase --gid 6014 && \ + useradd hbase --gid 6014 --uid 6014 -m diff --git a/build-image.sh b/build-image.sh index 9c74cea..b6f7410 100755 --- a/build-image.sh +++ b/build-image.sh @@ -73,6 +73,7 @@ function build_hadoop_master_image() { cp download/jcl-over-slf4j-1.7.36.jar hadoop-master${INDEX}/download/jcl-over-slf4j-1.7.36.jar cp download/trino-server-${TRINO_VERSION}.tar.gz hadoop-master${INDEX}/download/trino-server-${TRINO_VERSION}.tar.gz cp download/trino-cli-${TRINO_VERSION}-executable.jar hadoop-master${INDEX}/download/trino-cli-${TRINO_VERSION}-executable.jar + cp download/hbase-${HBASE_VERSION}-bin.tar.gz hadoop-master${INDEX}/download/hbase-${HBASE_VERSION}-bin.tar.gz ${BUILD_CMD} \ --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ --build-arg ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION} \ @@ -88,6 +89,7 @@ function build_hadoop_master_image() { --build-arg HUDI_VERSION=${HUDI_VERSION} \ --build-arg LOKI_APPENDER_VERSION=${LOKI_APPENDER_VERSION} \ --build-arg TRINO_VERSION=${TRINO_VERSION} \ + --build-arg HBASE_VERSION=${HBASE_VERSION} \ --file "${SELF_DIR}/hadoop-master${INDEX}/Dockerfile" \ --tag hadoop-testing/hadoop-master${INDEX}:${PROJECT_VERSION} \ "${SELF_DIR}/hadoop-master${INDEX}" $2 @@ -110,6 +112,7 @@ function build_hadoop_worker_image() { --build-arg HADOOP_VERSION=${HADOOP_VERSION} \ --build-arg SPARK_VERSION=${SPARK_VERSION} \ --build-arg TRINO_VERSION=${TRINO_VERSION} \ + --build-arg HBASE_VERSION=${HBASE_VERSION} \ --file "${SELF_DIR}/hadoop-worker${INDEX}/Dockerfile" \ --tag hadoop-testing/hadoop-worker${INDEX}:${PROJECT_VERSION} \ "${SELF_DIR}/hadoop-worker${INDEX}" $2 diff --git a/hadoop-master1/Dockerfile b/hadoop-master1/Dockerfile index d222ae6..2196106 100644 --- a/hadoop-master1/Dockerfile +++ b/hadoop-master1/Dockerfile @@ -26,6 +26,7 @@ ARG LOKI_APPENDER_VERSION ARG RANGER_VERSION ARG HUDI_VERSION ARG TRINO_VERSION +ARG HBASE_VERSION ENV ZOOKEEPER_HOME=/opt/zookeeper ENV ZOOCFGDIR=/etc/zookeeper/conf @@ -45,6 +46,7 @@ ENV TRINO_HOME=/opt/trino ENV RANGER_VERSION=${RANGER_VERSION} ENV HUDI_VERSION=${HUDI_VERSION} ENV MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION} +ENV HBASE_HOME=/opt/hbase ENV PATH=${KYUUBI_HOME}/bin:${SPARK_HOME}/bin:${HIVE_HOME}/bin:${HADOOP_HOME}/bin:${ZOOKEEPER_HOME}/bin:${PATH} ADD download/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz /opt @@ -55,6 +57,7 @@ ADD download/flink-${FLINK_VERSION}-bin-scala_${SCALA_BINARY_VERSION}.tgz /opt ADD download/apache-kyuubi-${KYUUBI_VERSION}-bin.tgz /opt ADD download/ranger-${RANGER_VERSION}-admin.tar.gz /opt ADD download/trino-server-${TRINO_VERSION}.tar.gz /opt +ADD download/hbase-${HBASE_VERSION}-bin.tar.gz /opt # Copy configuration files COPY ./files / @@ -66,7 +69,8 @@ RUN ln -snf /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin ${ZOOKEEPER_HOME} && ln -snf /opt/flink-${FLINK_VERSION} ${FLINK_HOME} && \ ln -snf /opt/apache-kyuubi-${KYUUBI_VERSION}-bin ${KYUUBI_HOME} && \ ln -snf /opt/ranger-${RANGER_VERSION}-admin ${RANGER_HOME} && \ - ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} + ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} && \ + ln -snf /opt/hbase-${HBASE_VERSION} ${HBASE_HOME} ADD download/mysql-connector-j-${MYSQL_JDBC_VERSION}.jar ${HIVE_HOME}/lib/ ADD download/mysql-connector-j-${MYSQL_JDBC_VERSION}.jar ${RANGER_HOME}/ @@ -90,7 +94,8 @@ RUN chown -R zookeeper:hadoop /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin && chown -R flink:hadoop /opt/flink-${FLINK_VERSION} && \ chown -R kyuubi:hadoop /opt/apache-kyuubi-${KYUUBI_VERSION}-bin && \ chown -R ranger:hadoop /opt/ranger-${RANGER_VERSION}-admin && \ - chown -R trino:hadoop /opt/trino-server-${TRINO_VERSION} + chown -R trino:hadoop /opt/trino-server-${TRINO_VERSION} && \ + chown -R hbase:hadoop /opt/hbase-${HBASE_VERSION} RUN chmod +x ${TRINO_HOME}/bin/trino-cli @@ -118,4 +123,7 @@ EXPOSE 6080 # Trino ports EXPOSE 18081 +# HBase ports +EXPOSE 16010 + CMD supervisord -c /etc/supervisord.conf diff --git a/hadoop-master1/files/etc/hbase/conf/hadoop-metrics2-hbase.properties b/hadoop-master1/files/etc/hbase/conf/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..4c7dbbe --- /dev/null +++ b/hadoop-master1/files/etc/hbase/conf/hadoop-metrics2-hbase.properties @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +*.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink +# default sampling period +*.period=10 + +# Below are some examples of sinks that could be used +# to monitor different hbase daemons. + +# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file-all.filename=all.metrics + +# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file0.context=hmaster +# hbase.sink.file0.filename=master.metrics + +# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file1.context=thrift-one +# hbase.sink.file1.filename=thrift-one.metrics + +# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file2.context=thrift-two +# hbase.sink.file2.filename=thrift-one.metrics + +# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file3.context=rest +# hbase.sink.file3.filename=rest.metrics diff --git a/hadoop-master1/files/etc/hbase/conf/hbase-env.sh b/hadoop-master1/files/etc/hbase/conf/hbase-env.sh new file mode 100644 index 0000000..3053de5 --- /dev/null +++ b/hadoop-master1/files/etc/hbase/conf/hbase-env.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# This script sets variables multiple times over the course of starting an hbase process, +# so try to keep things idempotent unless you want to take an even deeper look +# into the startup scripts (bin/hbase, etc.) + +# The java implementation to use. Java 1.8+ required. +export JAVA_HOME=/opt/openjdk-8 + +# Extra Java CLASSPATH elements. Optional. +# export HBASE_CLASSPATH= + +# The maximum amount of heap to use. Default is left to JVM default. +# export HBASE_HEAPSIZE=1G + +# Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +# offheap, set the value to "8G". +# export HBASE_OFFHEAPSIZE=1G + +# Extra Java runtime options. +# Default settings are applied according to the detected JVM version. Override these default +# settings by specifying a value here. For more details on possible settings, +# see http://hbase.apache.org/book.html#_jvm_tuning +# export HBASE_OPTS + +# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. + +# This enables basic gc logging to the .out file. +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment one of the below three options to enable java garbage collection logging for the client processes. + +# This enables basic gc logging to the .out file. +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations +# needed setting up off-heap block caching. + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX +# section in HBase Reference Guide for instructions. + +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# Uncomment and adjust to keep all the Region Server pages mapped to be memory resident +#HBASE_REGIONSERVER_MLOCK=true +#HBASE_REGIONSERVER_UID="hbase" + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8074" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of ZooKeeper or not. +export HBASE_MANAGES_ZK=false + +# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the +# RFA appender. Please refer to the log4j2.properties file to see more details on this appender. +# In case one needs to do log rolling on a date change, one should set the environment property +# HBASE_ROOT_LOGGER to ",DRFA". +# For example: +# export HBASE_ROOT_LOGGER=INFO,DRFA +# The reason for changing default to RFA is to avoid the boundary case of filling out disk space as +# DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. + +# Tell HBase whether it should include Hadoop's lib when start up, +# the default value is false,means that includes Hadoop's lib. +# export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true" + +# Override text processing tools for use by these launch scripts. +# export GREP="${GREP-grep}" +# export SED="${SED-sed}" + +# +## OpenTelemetry Tracing +# +# HBase is instrumented for tracing using OpenTelemetry. None of the other OpenTelemetry signals +# are supported at this time. Configuring tracing involves setting several configuration points, +# via environment variable or system property. This configuration prefers setting environment +# variables whenever possible because they are picked up by all processes launched by `bin/hbase`. +# Use system properties when you launch multiple processes from the same configuration directory -- +# when you need to specify different configuration values for different hbase processes that are +# launched using the same HBase configuration (i.e., a single-host pseudo-distributed cluster or +# launching the `bin/hbase shell` from a host that is also running an instance of the master). See +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure +# for an inventory of configuration points and detailed explanations of each of them. +# +# Note also that as of this writing, the javaagent logs to stderr and is not configured along with +# the rest of HBase's logging configuration. +# +# `HBASE_OTEL_TRACING_ENABLED`, required. Enable attaching the opentelemetry javaagent to the +# process via support provided by `bin/hbase`. When this value us `false`, the agent is not added +# to the process launch arguments and all further OpenTelemetry configuration is ignored. +#export HBASE_OTEL_TRACING_ENABLED=true +# +# `OPENTELEMETRY_JAVAAGENT_PATH`, optional. Override the javaagent provided by HBase in `lib/trace` +# with an alternate. Use when you need to upgrade the agent version or swap out the official one +# for an alternative implementation. +#export OPENTELEMETRY_JAVAAGENT_PATH="" +# +# `OTEL_FOO_EXPORTER`, required. Specify an Exporter implementation per signal type. HBase only +# makes explicit use of the traces signal at this time, so the important one is +# `OTEL_TRACES_EXPORTER`. Specify its value based on the exporter required for your tracing +# environment. The other two should be uncommented and specified as `none`, otherwise the agent +# may report errors while attempting to export these other signals to an unconfigured destination. +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#exporters +#export OTEL_TRACES_EXPORTER="" +#export OTEL_METRICS_EXPORTER="none" +#export OTEL_LOGS_EXPORTER="none" +# +# `OTEL_SERVICE_NAME`, required. Specify "resource attributes", and specifically the `service.name`, +# as a unique value for each HBase process. OpenTelemetry allows for specifying this value in one +# of two ways, via environment variables with the `OTEL_` prefix, or via system properties with the +# `otel.` prefix. Which you use with HBase is decided based on whether this configuration file is +# read by a single process or shared by multiple HBase processes. For the default standalone mode +# or an environment where all processes share the same configuration file, use the `otel` system +# properties by uncommenting all of the `HBASE_FOO_OPTS` exports below. When this configuration file +# is being consumed by only a single process -- for example, from a systemd configuration or in a +# container template -- replace use of `HBASE_FOO_OPTS` with the standard `OTEL_SERVICE_NAME` and/or +# `OTEL_RESOURCE_ATTRIBUTES` environment variables. For further details, see +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#opentelemetry-resource +#export HBASE_CANARY_OPTS="${HBASE_CANARY_OPTS} -Dotel.resource.attributes=service.name=hbase-canary" +#export HBASE_HBCK_OPTS="${HBASE_HBCK_OPTS} -Dotel.resource.attributes=service.name=hbase-hbck" +#export HBASE_HBTOP_OPTS="${HBASE_HBTOP_OPTS} -Dotel.resource.attributes=service.name=hbase-hbtop" +#export HBASE_JSHELL_OPTS="${HBASE_JSHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-jshell" +#export HBASE_LTT_OPTS="${HBASE_LTT_OPTS} -Dotel.resource.attributes=service.name=hbase-loadtesttool" +#export HBASE_MASTER_OPTS="${HBASE_MASTER_OPTS} -Dotel.resource.attributes=service.name=hbase-master" +#export HBASE_PE_OPTS="${HBASE_PE_OPTS} -Dotel.resource.attributes=service.name=hbase-performanceevaluation" +#export HBASE_REGIONSERVER_OPTS="${HBASE_REGIONSERVER_OPTS} -Dotel.resource.attributes=service.name=hbase-regionserver" +#export HBASE_REST_OPTS="${HBASE_REST_OPTS} -Dotel.resource.attributes=service.name=hbase-rest" +#export HBASE_SHELL_OPTS="${HBASE_SHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-shell" +#export HBASE_THRIFT_OPTS="${HBASE_THRIFT_OPTS} -Dotel.resource.attributes=service.name=hbase-thrift" +#export HBASE_ZOOKEEPER_OPTS="${HBASE_ZOOKEEPER_OPTS} -Dotel.resource.attributes=service.name=hbase-zookeeper" + +# +# JDK11+ JShell +# +# Additional arguments passed to jshell invocation +# export HBASE_JSHELL_ARGS="--startup DEFAULT --startup PRINTING --startup hbase_startup.jsh" diff --git a/hadoop-master1/files/etc/hbase/conf/hbase-policy.xml b/hadoop-master1/files/etc/hbase/conf/hbase-policy.xml new file mode 100644 index 0000000..5a0256d --- /dev/null +++ b/hadoop-master1/files/etc/hbase/conf/hbase-policy.xml @@ -0,0 +1,53 @@ + + + + + + + security.client.protocol.acl + * + ACL for ClientProtocol and AdminProtocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.admin.protocol.acl + * + ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.masterregion.protocol.acl + * + ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + diff --git a/hadoop-master1/files/etc/hbase/conf/hbase-site.xml b/hadoop-master1/files/etc/hbase/conf/hbase-site.xml new file mode 100644 index 0000000..a8f52a1 --- /dev/null +++ b/hadoop-master1/files/etc/hbase/conf/hbase-site.xml @@ -0,0 +1,70 @@ + + + + + + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://hadoop-master1.orb.local:8020/hbase + + + hbase.tmp.dir + hdfs://hadoop-master1.orb.local:8020/hbase/tmp + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.zookeeper.quorum + hadoop-master1.orb.local + + + hbase.zookeeper.property.clientPort + 2181 + + + hbase.master.info.port + 16010 + + diff --git a/hadoop-master1/files/etc/hbase/conf/log4j2-hbtop.properties b/hadoop-master1/files/etc/hbase/conf/log4j2-hbtop.properties new file mode 100644 index 0000000..de2f976 --- /dev/null +++ b/hadoop-master1/files/etc/hbase/conf/log4j2-hbtop.properties @@ -0,0 +1,35 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# console +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %m%n + +rootLogger = WARN,console + +# ZooKeeper will still put stuff at WARN +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = ERROR + diff --git a/hadoop-master1/files/etc/hbase/conf/log4j2.properties b/hadoop-master1/files/etc/hbase/conf/log4j2.properties new file mode 100644 index 0000000..5ffcfda --- /dev/null +++ b/hadoop-master1/files/etc/hbase/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +# logger.hbase.name = org.apache.hadoop.hbase +# logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/hadoop-master1/files/etc/hbase/conf/regionservers b/hadoop-master1/files/etc/hbase/conf/regionservers new file mode 100644 index 0000000..ee398e3 --- /dev/null +++ b/hadoop-master1/files/etc/hbase/conf/regionservers @@ -0,0 +1,3 @@ +hadoop-worker1.orb.local +hadoop-worker2.orb.local +hadoop-worker3.orb.local \ No newline at end of file diff --git a/hadoop-worker1/Dockerfile b/hadoop-worker1/Dockerfile index b813a05..a7580ec 100644 --- a/hadoop-worker1/Dockerfile +++ b/hadoop-worker1/Dockerfile @@ -16,23 +16,27 @@ FROM hadoop-testing/base-ubuntu-2204:$PROJECT_VERSION ARG HADOOP_VERSION ARG SPARK_VERSION ARG TRINO_VERSION +ARG HBASE_VERSION ENV HADOOP_HOME=/opt/hadoop ENV HADOOP_CONF_DIR=/etc/hadoop/conf ENV HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/client/* ENV TRINO_HOME=/opt/trino +ENV HBASE_HOME=/opt/hbase ENV PATH=${HADOOP_HOME}/bin:${PATH} ADD download/hadoop-${HADOOP_VERSION}.tar.gz /opt ADD download/spark-${SPARK_VERSION}-bin-hadoop3/yarn /opt/spark-${SPARK_VERSION}-bin-hadoop3/yarn ADD download/trino-server-${TRINO_VERSION}.tar.gz /opt +ADD download/hbase-${HBASE_VERSION}-bin.tar.gz /opt # Copy configuration files COPY ./files / RUN ln -snf /opt/hadoop-${HADOOP_VERSION} ${HADOOP_HOME} && \ ln -snf spark-${SPARK_VERSION}-bin-hadoop3 /opt/spark && \ - ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} + ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} && \ + ln -snf /opt/hbase-${HBASE_VERSION} ${HBASE_HOME} RUN /opt/hadoop-init.d/init-hdfs.sh RUN /opt/trino-init.d/init-workdir.sh diff --git a/hadoop-worker1/files/etc/hbase/conf/hadoop-metrics2-hbase.properties b/hadoop-worker1/files/etc/hbase/conf/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..4c7dbbe --- /dev/null +++ b/hadoop-worker1/files/etc/hbase/conf/hadoop-metrics2-hbase.properties @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +*.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink +# default sampling period +*.period=10 + +# Below are some examples of sinks that could be used +# to monitor different hbase daemons. + +# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file-all.filename=all.metrics + +# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file0.context=hmaster +# hbase.sink.file0.filename=master.metrics + +# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file1.context=thrift-one +# hbase.sink.file1.filename=thrift-one.metrics + +# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file2.context=thrift-two +# hbase.sink.file2.filename=thrift-one.metrics + +# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file3.context=rest +# hbase.sink.file3.filename=rest.metrics diff --git a/hadoop-worker1/files/etc/hbase/conf/hbase-env.sh b/hadoop-worker1/files/etc/hbase/conf/hbase-env.sh new file mode 100644 index 0000000..3053de5 --- /dev/null +++ b/hadoop-worker1/files/etc/hbase/conf/hbase-env.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# This script sets variables multiple times over the course of starting an hbase process, +# so try to keep things idempotent unless you want to take an even deeper look +# into the startup scripts (bin/hbase, etc.) + +# The java implementation to use. Java 1.8+ required. +export JAVA_HOME=/opt/openjdk-8 + +# Extra Java CLASSPATH elements. Optional. +# export HBASE_CLASSPATH= + +# The maximum amount of heap to use. Default is left to JVM default. +# export HBASE_HEAPSIZE=1G + +# Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +# offheap, set the value to "8G". +# export HBASE_OFFHEAPSIZE=1G + +# Extra Java runtime options. +# Default settings are applied according to the detected JVM version. Override these default +# settings by specifying a value here. For more details on possible settings, +# see http://hbase.apache.org/book.html#_jvm_tuning +# export HBASE_OPTS + +# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. + +# This enables basic gc logging to the .out file. +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment one of the below three options to enable java garbage collection logging for the client processes. + +# This enables basic gc logging to the .out file. +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations +# needed setting up off-heap block caching. + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX +# section in HBase Reference Guide for instructions. + +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# Uncomment and adjust to keep all the Region Server pages mapped to be memory resident +#HBASE_REGIONSERVER_MLOCK=true +#HBASE_REGIONSERVER_UID="hbase" + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8074" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of ZooKeeper or not. +export HBASE_MANAGES_ZK=false + +# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the +# RFA appender. Please refer to the log4j2.properties file to see more details on this appender. +# In case one needs to do log rolling on a date change, one should set the environment property +# HBASE_ROOT_LOGGER to ",DRFA". +# For example: +# export HBASE_ROOT_LOGGER=INFO,DRFA +# The reason for changing default to RFA is to avoid the boundary case of filling out disk space as +# DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. + +# Tell HBase whether it should include Hadoop's lib when start up, +# the default value is false,means that includes Hadoop's lib. +# export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true" + +# Override text processing tools for use by these launch scripts. +# export GREP="${GREP-grep}" +# export SED="${SED-sed}" + +# +## OpenTelemetry Tracing +# +# HBase is instrumented for tracing using OpenTelemetry. None of the other OpenTelemetry signals +# are supported at this time. Configuring tracing involves setting several configuration points, +# via environment variable or system property. This configuration prefers setting environment +# variables whenever possible because they are picked up by all processes launched by `bin/hbase`. +# Use system properties when you launch multiple processes from the same configuration directory -- +# when you need to specify different configuration values for different hbase processes that are +# launched using the same HBase configuration (i.e., a single-host pseudo-distributed cluster or +# launching the `bin/hbase shell` from a host that is also running an instance of the master). See +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure +# for an inventory of configuration points and detailed explanations of each of them. +# +# Note also that as of this writing, the javaagent logs to stderr and is not configured along with +# the rest of HBase's logging configuration. +# +# `HBASE_OTEL_TRACING_ENABLED`, required. Enable attaching the opentelemetry javaagent to the +# process via support provided by `bin/hbase`. When this value us `false`, the agent is not added +# to the process launch arguments and all further OpenTelemetry configuration is ignored. +#export HBASE_OTEL_TRACING_ENABLED=true +# +# `OPENTELEMETRY_JAVAAGENT_PATH`, optional. Override the javaagent provided by HBase in `lib/trace` +# with an alternate. Use when you need to upgrade the agent version or swap out the official one +# for an alternative implementation. +#export OPENTELEMETRY_JAVAAGENT_PATH="" +# +# `OTEL_FOO_EXPORTER`, required. Specify an Exporter implementation per signal type. HBase only +# makes explicit use of the traces signal at this time, so the important one is +# `OTEL_TRACES_EXPORTER`. Specify its value based on the exporter required for your tracing +# environment. The other two should be uncommented and specified as `none`, otherwise the agent +# may report errors while attempting to export these other signals to an unconfigured destination. +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#exporters +#export OTEL_TRACES_EXPORTER="" +#export OTEL_METRICS_EXPORTER="none" +#export OTEL_LOGS_EXPORTER="none" +# +# `OTEL_SERVICE_NAME`, required. Specify "resource attributes", and specifically the `service.name`, +# as a unique value for each HBase process. OpenTelemetry allows for specifying this value in one +# of two ways, via environment variables with the `OTEL_` prefix, or via system properties with the +# `otel.` prefix. Which you use with HBase is decided based on whether this configuration file is +# read by a single process or shared by multiple HBase processes. For the default standalone mode +# or an environment where all processes share the same configuration file, use the `otel` system +# properties by uncommenting all of the `HBASE_FOO_OPTS` exports below. When this configuration file +# is being consumed by only a single process -- for example, from a systemd configuration or in a +# container template -- replace use of `HBASE_FOO_OPTS` with the standard `OTEL_SERVICE_NAME` and/or +# `OTEL_RESOURCE_ATTRIBUTES` environment variables. For further details, see +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#opentelemetry-resource +#export HBASE_CANARY_OPTS="${HBASE_CANARY_OPTS} -Dotel.resource.attributes=service.name=hbase-canary" +#export HBASE_HBCK_OPTS="${HBASE_HBCK_OPTS} -Dotel.resource.attributes=service.name=hbase-hbck" +#export HBASE_HBTOP_OPTS="${HBASE_HBTOP_OPTS} -Dotel.resource.attributes=service.name=hbase-hbtop" +#export HBASE_JSHELL_OPTS="${HBASE_JSHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-jshell" +#export HBASE_LTT_OPTS="${HBASE_LTT_OPTS} -Dotel.resource.attributes=service.name=hbase-loadtesttool" +#export HBASE_MASTER_OPTS="${HBASE_MASTER_OPTS} -Dotel.resource.attributes=service.name=hbase-master" +#export HBASE_PE_OPTS="${HBASE_PE_OPTS} -Dotel.resource.attributes=service.name=hbase-performanceevaluation" +#export HBASE_REGIONSERVER_OPTS="${HBASE_REGIONSERVER_OPTS} -Dotel.resource.attributes=service.name=hbase-regionserver" +#export HBASE_REST_OPTS="${HBASE_REST_OPTS} -Dotel.resource.attributes=service.name=hbase-rest" +#export HBASE_SHELL_OPTS="${HBASE_SHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-shell" +#export HBASE_THRIFT_OPTS="${HBASE_THRIFT_OPTS} -Dotel.resource.attributes=service.name=hbase-thrift" +#export HBASE_ZOOKEEPER_OPTS="${HBASE_ZOOKEEPER_OPTS} -Dotel.resource.attributes=service.name=hbase-zookeeper" + +# +# JDK11+ JShell +# +# Additional arguments passed to jshell invocation +# export HBASE_JSHELL_ARGS="--startup DEFAULT --startup PRINTING --startup hbase_startup.jsh" diff --git a/hadoop-worker1/files/etc/hbase/conf/hbase-policy.xml b/hadoop-worker1/files/etc/hbase/conf/hbase-policy.xml new file mode 100644 index 0000000..5a0256d --- /dev/null +++ b/hadoop-worker1/files/etc/hbase/conf/hbase-policy.xml @@ -0,0 +1,53 @@ + + + + + + + security.client.protocol.acl + * + ACL for ClientProtocol and AdminProtocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.admin.protocol.acl + * + ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.masterregion.protocol.acl + * + ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + diff --git a/hadoop-worker1/files/etc/hbase/conf/hbase-site.xml b/hadoop-worker1/files/etc/hbase/conf/hbase-site.xml new file mode 100644 index 0000000..a8f52a1 --- /dev/null +++ b/hadoop-worker1/files/etc/hbase/conf/hbase-site.xml @@ -0,0 +1,70 @@ + + + + + + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://hadoop-master1.orb.local:8020/hbase + + + hbase.tmp.dir + hdfs://hadoop-master1.orb.local:8020/hbase/tmp + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.zookeeper.quorum + hadoop-master1.orb.local + + + hbase.zookeeper.property.clientPort + 2181 + + + hbase.master.info.port + 16010 + + diff --git a/hadoop-worker1/files/etc/hbase/conf/log4j2-hbtop.properties b/hadoop-worker1/files/etc/hbase/conf/log4j2-hbtop.properties new file mode 100644 index 0000000..de2f976 --- /dev/null +++ b/hadoop-worker1/files/etc/hbase/conf/log4j2-hbtop.properties @@ -0,0 +1,35 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# console +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %m%n + +rootLogger = WARN,console + +# ZooKeeper will still put stuff at WARN +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = ERROR + diff --git a/hadoop-worker1/files/etc/hbase/conf/log4j2.properties b/hadoop-worker1/files/etc/hbase/conf/log4j2.properties new file mode 100644 index 0000000..5ffcfda --- /dev/null +++ b/hadoop-worker1/files/etc/hbase/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +# logger.hbase.name = org.apache.hadoop.hbase +# logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/hadoop-worker1/files/etc/hbase/conf/regionservers b/hadoop-worker1/files/etc/hbase/conf/regionservers new file mode 100644 index 0000000..ee398e3 --- /dev/null +++ b/hadoop-worker1/files/etc/hbase/conf/regionservers @@ -0,0 +1,3 @@ +hadoop-worker1.orb.local +hadoop-worker2.orb.local +hadoop-worker3.orb.local \ No newline at end of file diff --git a/hadoop-worker2/Dockerfile b/hadoop-worker2/Dockerfile index b813a05..1c9141c 100644 --- a/hadoop-worker2/Dockerfile +++ b/hadoop-worker2/Dockerfile @@ -16,11 +16,13 @@ FROM hadoop-testing/base-ubuntu-2204:$PROJECT_VERSION ARG HADOOP_VERSION ARG SPARK_VERSION ARG TRINO_VERSION +ARG HBASE_VERSION ENV HADOOP_HOME=/opt/hadoop ENV HADOOP_CONF_DIR=/etc/hadoop/conf ENV HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/client/* ENV TRINO_HOME=/opt/trino +ENV HBASE_HOME=/opt/hbase ENV PATH=${HADOOP_HOME}/bin:${PATH} ADD download/hadoop-${HADOOP_VERSION}.tar.gz /opt @@ -32,14 +34,16 @@ COPY ./files / RUN ln -snf /opt/hadoop-${HADOOP_VERSION} ${HADOOP_HOME} && \ ln -snf spark-${SPARK_VERSION}-bin-hadoop3 /opt/spark && \ - ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} + ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} && \ + ln -snf /opt/hbase-${HBASE_VERSION} ${HBASE_HOME} RUN /opt/hadoop-init.d/init-hdfs.sh RUN /opt/trino-init.d/init-workdir.sh RUN chown -R hadoop:hadoop /opt/hadoop-${HADOOP_VERSION} && \ chown -R spark:hadoop /opt/spark-${SPARK_VERSION}-bin-hadoop3 && \ - chown -R trino:hadoop /opt/trino-server-${TRINO_VERSION} + chown -R trino:hadoop /opt/trino-server-${TRINO_VERSION} && \ + chown -R hbase:hadoop /opt/hbase-${HBASE_VERSION} # HDFS ports EXPOSE 1004 1006 8020 9866 9867 9870 9864 50470 diff --git a/hadoop-worker2/files/etc/hbase/conf/hadoop-metrics2-hbase.properties b/hadoop-worker2/files/etc/hbase/conf/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..4c7dbbe --- /dev/null +++ b/hadoop-worker2/files/etc/hbase/conf/hadoop-metrics2-hbase.properties @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +*.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink +# default sampling period +*.period=10 + +# Below are some examples of sinks that could be used +# to monitor different hbase daemons. + +# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file-all.filename=all.metrics + +# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file0.context=hmaster +# hbase.sink.file0.filename=master.metrics + +# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file1.context=thrift-one +# hbase.sink.file1.filename=thrift-one.metrics + +# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file2.context=thrift-two +# hbase.sink.file2.filename=thrift-one.metrics + +# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file3.context=rest +# hbase.sink.file3.filename=rest.metrics diff --git a/hadoop-worker2/files/etc/hbase/conf/hbase-env.sh b/hadoop-worker2/files/etc/hbase/conf/hbase-env.sh new file mode 100644 index 0000000..3053de5 --- /dev/null +++ b/hadoop-worker2/files/etc/hbase/conf/hbase-env.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# This script sets variables multiple times over the course of starting an hbase process, +# so try to keep things idempotent unless you want to take an even deeper look +# into the startup scripts (bin/hbase, etc.) + +# The java implementation to use. Java 1.8+ required. +export JAVA_HOME=/opt/openjdk-8 + +# Extra Java CLASSPATH elements. Optional. +# export HBASE_CLASSPATH= + +# The maximum amount of heap to use. Default is left to JVM default. +# export HBASE_HEAPSIZE=1G + +# Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +# offheap, set the value to "8G". +# export HBASE_OFFHEAPSIZE=1G + +# Extra Java runtime options. +# Default settings are applied according to the detected JVM version. Override these default +# settings by specifying a value here. For more details on possible settings, +# see http://hbase.apache.org/book.html#_jvm_tuning +# export HBASE_OPTS + +# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. + +# This enables basic gc logging to the .out file. +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment one of the below three options to enable java garbage collection logging for the client processes. + +# This enables basic gc logging to the .out file. +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations +# needed setting up off-heap block caching. + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX +# section in HBase Reference Guide for instructions. + +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# Uncomment and adjust to keep all the Region Server pages mapped to be memory resident +#HBASE_REGIONSERVER_MLOCK=true +#HBASE_REGIONSERVER_UID="hbase" + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8074" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of ZooKeeper or not. +export HBASE_MANAGES_ZK=false + +# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the +# RFA appender. Please refer to the log4j2.properties file to see more details on this appender. +# In case one needs to do log rolling on a date change, one should set the environment property +# HBASE_ROOT_LOGGER to ",DRFA". +# For example: +# export HBASE_ROOT_LOGGER=INFO,DRFA +# The reason for changing default to RFA is to avoid the boundary case of filling out disk space as +# DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. + +# Tell HBase whether it should include Hadoop's lib when start up, +# the default value is false,means that includes Hadoop's lib. +# export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true" + +# Override text processing tools for use by these launch scripts. +# export GREP="${GREP-grep}" +# export SED="${SED-sed}" + +# +## OpenTelemetry Tracing +# +# HBase is instrumented for tracing using OpenTelemetry. None of the other OpenTelemetry signals +# are supported at this time. Configuring tracing involves setting several configuration points, +# via environment variable or system property. This configuration prefers setting environment +# variables whenever possible because they are picked up by all processes launched by `bin/hbase`. +# Use system properties when you launch multiple processes from the same configuration directory -- +# when you need to specify different configuration values for different hbase processes that are +# launched using the same HBase configuration (i.e., a single-host pseudo-distributed cluster or +# launching the `bin/hbase shell` from a host that is also running an instance of the master). See +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure +# for an inventory of configuration points and detailed explanations of each of them. +# +# Note also that as of this writing, the javaagent logs to stderr and is not configured along with +# the rest of HBase's logging configuration. +# +# `HBASE_OTEL_TRACING_ENABLED`, required. Enable attaching the opentelemetry javaagent to the +# process via support provided by `bin/hbase`. When this value us `false`, the agent is not added +# to the process launch arguments and all further OpenTelemetry configuration is ignored. +#export HBASE_OTEL_TRACING_ENABLED=true +# +# `OPENTELEMETRY_JAVAAGENT_PATH`, optional. Override the javaagent provided by HBase in `lib/trace` +# with an alternate. Use when you need to upgrade the agent version or swap out the official one +# for an alternative implementation. +#export OPENTELEMETRY_JAVAAGENT_PATH="" +# +# `OTEL_FOO_EXPORTER`, required. Specify an Exporter implementation per signal type. HBase only +# makes explicit use of the traces signal at this time, so the important one is +# `OTEL_TRACES_EXPORTER`. Specify its value based on the exporter required for your tracing +# environment. The other two should be uncommented and specified as `none`, otherwise the agent +# may report errors while attempting to export these other signals to an unconfigured destination. +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#exporters +#export OTEL_TRACES_EXPORTER="" +#export OTEL_METRICS_EXPORTER="none" +#export OTEL_LOGS_EXPORTER="none" +# +# `OTEL_SERVICE_NAME`, required. Specify "resource attributes", and specifically the `service.name`, +# as a unique value for each HBase process. OpenTelemetry allows for specifying this value in one +# of two ways, via environment variables with the `OTEL_` prefix, or via system properties with the +# `otel.` prefix. Which you use with HBase is decided based on whether this configuration file is +# read by a single process or shared by multiple HBase processes. For the default standalone mode +# or an environment where all processes share the same configuration file, use the `otel` system +# properties by uncommenting all of the `HBASE_FOO_OPTS` exports below. When this configuration file +# is being consumed by only a single process -- for example, from a systemd configuration or in a +# container template -- replace use of `HBASE_FOO_OPTS` with the standard `OTEL_SERVICE_NAME` and/or +# `OTEL_RESOURCE_ATTRIBUTES` environment variables. For further details, see +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#opentelemetry-resource +#export HBASE_CANARY_OPTS="${HBASE_CANARY_OPTS} -Dotel.resource.attributes=service.name=hbase-canary" +#export HBASE_HBCK_OPTS="${HBASE_HBCK_OPTS} -Dotel.resource.attributes=service.name=hbase-hbck" +#export HBASE_HBTOP_OPTS="${HBASE_HBTOP_OPTS} -Dotel.resource.attributes=service.name=hbase-hbtop" +#export HBASE_JSHELL_OPTS="${HBASE_JSHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-jshell" +#export HBASE_LTT_OPTS="${HBASE_LTT_OPTS} -Dotel.resource.attributes=service.name=hbase-loadtesttool" +#export HBASE_MASTER_OPTS="${HBASE_MASTER_OPTS} -Dotel.resource.attributes=service.name=hbase-master" +#export HBASE_PE_OPTS="${HBASE_PE_OPTS} -Dotel.resource.attributes=service.name=hbase-performanceevaluation" +#export HBASE_REGIONSERVER_OPTS="${HBASE_REGIONSERVER_OPTS} -Dotel.resource.attributes=service.name=hbase-regionserver" +#export HBASE_REST_OPTS="${HBASE_REST_OPTS} -Dotel.resource.attributes=service.name=hbase-rest" +#export HBASE_SHELL_OPTS="${HBASE_SHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-shell" +#export HBASE_THRIFT_OPTS="${HBASE_THRIFT_OPTS} -Dotel.resource.attributes=service.name=hbase-thrift" +#export HBASE_ZOOKEEPER_OPTS="${HBASE_ZOOKEEPER_OPTS} -Dotel.resource.attributes=service.name=hbase-zookeeper" + +# +# JDK11+ JShell +# +# Additional arguments passed to jshell invocation +# export HBASE_JSHELL_ARGS="--startup DEFAULT --startup PRINTING --startup hbase_startup.jsh" diff --git a/hadoop-worker2/files/etc/hbase/conf/hbase-policy.xml b/hadoop-worker2/files/etc/hbase/conf/hbase-policy.xml new file mode 100644 index 0000000..5a0256d --- /dev/null +++ b/hadoop-worker2/files/etc/hbase/conf/hbase-policy.xml @@ -0,0 +1,53 @@ + + + + + + + security.client.protocol.acl + * + ACL for ClientProtocol and AdminProtocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.admin.protocol.acl + * + ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.masterregion.protocol.acl + * + ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + diff --git a/hadoop-worker2/files/etc/hbase/conf/hbase-site.xml b/hadoop-worker2/files/etc/hbase/conf/hbase-site.xml new file mode 100644 index 0000000..a8f52a1 --- /dev/null +++ b/hadoop-worker2/files/etc/hbase/conf/hbase-site.xml @@ -0,0 +1,70 @@ + + + + + + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://hadoop-master1.orb.local:8020/hbase + + + hbase.tmp.dir + hdfs://hadoop-master1.orb.local:8020/hbase/tmp + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.zookeeper.quorum + hadoop-master1.orb.local + + + hbase.zookeeper.property.clientPort + 2181 + + + hbase.master.info.port + 16010 + + diff --git a/hadoop-worker2/files/etc/hbase/conf/log4j2-hbtop.properties b/hadoop-worker2/files/etc/hbase/conf/log4j2-hbtop.properties new file mode 100644 index 0000000..de2f976 --- /dev/null +++ b/hadoop-worker2/files/etc/hbase/conf/log4j2-hbtop.properties @@ -0,0 +1,35 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# console +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %m%n + +rootLogger = WARN,console + +# ZooKeeper will still put stuff at WARN +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = ERROR + diff --git a/hadoop-worker2/files/etc/hbase/conf/log4j2.properties b/hadoop-worker2/files/etc/hbase/conf/log4j2.properties new file mode 100644 index 0000000..5ffcfda --- /dev/null +++ b/hadoop-worker2/files/etc/hbase/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +# logger.hbase.name = org.apache.hadoop.hbase +# logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/hadoop-worker2/files/etc/hbase/conf/regionservers b/hadoop-worker2/files/etc/hbase/conf/regionservers new file mode 100644 index 0000000..ee398e3 --- /dev/null +++ b/hadoop-worker2/files/etc/hbase/conf/regionservers @@ -0,0 +1,3 @@ +hadoop-worker1.orb.local +hadoop-worker2.orb.local +hadoop-worker3.orb.local \ No newline at end of file diff --git a/hadoop-worker3/Dockerfile b/hadoop-worker3/Dockerfile index b813a05..1c9141c 100644 --- a/hadoop-worker3/Dockerfile +++ b/hadoop-worker3/Dockerfile @@ -16,11 +16,13 @@ FROM hadoop-testing/base-ubuntu-2204:$PROJECT_VERSION ARG HADOOP_VERSION ARG SPARK_VERSION ARG TRINO_VERSION +ARG HBASE_VERSION ENV HADOOP_HOME=/opt/hadoop ENV HADOOP_CONF_DIR=/etc/hadoop/conf ENV HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/client/* ENV TRINO_HOME=/opt/trino +ENV HBASE_HOME=/opt/hbase ENV PATH=${HADOOP_HOME}/bin:${PATH} ADD download/hadoop-${HADOOP_VERSION}.tar.gz /opt @@ -32,14 +34,16 @@ COPY ./files / RUN ln -snf /opt/hadoop-${HADOOP_VERSION} ${HADOOP_HOME} && \ ln -snf spark-${SPARK_VERSION}-bin-hadoop3 /opt/spark && \ - ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} + ln -snf /opt/trino-server-${TRINO_VERSION} ${TRINO_HOME} && \ + ln -snf /opt/hbase-${HBASE_VERSION} ${HBASE_HOME} RUN /opt/hadoop-init.d/init-hdfs.sh RUN /opt/trino-init.d/init-workdir.sh RUN chown -R hadoop:hadoop /opt/hadoop-${HADOOP_VERSION} && \ chown -R spark:hadoop /opt/spark-${SPARK_VERSION}-bin-hadoop3 && \ - chown -R trino:hadoop /opt/trino-server-${TRINO_VERSION} + chown -R trino:hadoop /opt/trino-server-${TRINO_VERSION} && \ + chown -R hbase:hadoop /opt/hbase-${HBASE_VERSION} # HDFS ports EXPOSE 1004 1006 8020 9866 9867 9870 9864 50470 diff --git a/hadoop-worker3/files/etc/hbase/conf/hadoop-metrics2-hbase.properties b/hadoop-worker3/files/etc/hbase/conf/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..4c7dbbe --- /dev/null +++ b/hadoop-worker3/files/etc/hbase/conf/hadoop-metrics2-hbase.properties @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +*.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink +# default sampling period +*.period=10 + +# Below are some examples of sinks that could be used +# to monitor different hbase daemons. + +# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file-all.filename=all.metrics + +# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file0.context=hmaster +# hbase.sink.file0.filename=master.metrics + +# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file1.context=thrift-one +# hbase.sink.file1.filename=thrift-one.metrics + +# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file2.context=thrift-two +# hbase.sink.file2.filename=thrift-one.metrics + +# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file3.context=rest +# hbase.sink.file3.filename=rest.metrics diff --git a/hadoop-worker3/files/etc/hbase/conf/hbase-env.sh b/hadoop-worker3/files/etc/hbase/conf/hbase-env.sh new file mode 100644 index 0000000..3053de5 --- /dev/null +++ b/hadoop-worker3/files/etc/hbase/conf/hbase-env.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# This script sets variables multiple times over the course of starting an hbase process, +# so try to keep things idempotent unless you want to take an even deeper look +# into the startup scripts (bin/hbase, etc.) + +# The java implementation to use. Java 1.8+ required. +export JAVA_HOME=/opt/openjdk-8 + +# Extra Java CLASSPATH elements. Optional. +# export HBASE_CLASSPATH= + +# The maximum amount of heap to use. Default is left to JVM default. +# export HBASE_HEAPSIZE=1G + +# Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +# offheap, set the value to "8G". +# export HBASE_OFFHEAPSIZE=1G + +# Extra Java runtime options. +# Default settings are applied according to the detected JVM version. Override these default +# settings by specifying a value here. For more details on possible settings, +# see http://hbase.apache.org/book.html#_jvm_tuning +# export HBASE_OPTS + +# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. + +# This enables basic gc logging to the .out file. +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment one of the below three options to enable java garbage collection logging for the client processes. + +# This enables basic gc logging to the .out file. +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations +# needed setting up off-heap block caching. + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX +# section in HBase Reference Guide for instructions. + +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# Uncomment and adjust to keep all the Region Server pages mapped to be memory resident +#HBASE_REGIONSERVER_MLOCK=true +#HBASE_REGIONSERVER_UID="hbase" + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8074" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of ZooKeeper or not. +export HBASE_MANAGES_ZK=false + +# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the +# RFA appender. Please refer to the log4j2.properties file to see more details on this appender. +# In case one needs to do log rolling on a date change, one should set the environment property +# HBASE_ROOT_LOGGER to ",DRFA". +# For example: +# export HBASE_ROOT_LOGGER=INFO,DRFA +# The reason for changing default to RFA is to avoid the boundary case of filling out disk space as +# DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. + +# Tell HBase whether it should include Hadoop's lib when start up, +# the default value is false,means that includes Hadoop's lib. +# export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true" + +# Override text processing tools for use by these launch scripts. +# export GREP="${GREP-grep}" +# export SED="${SED-sed}" + +# +## OpenTelemetry Tracing +# +# HBase is instrumented for tracing using OpenTelemetry. None of the other OpenTelemetry signals +# are supported at this time. Configuring tracing involves setting several configuration points, +# via environment variable or system property. This configuration prefers setting environment +# variables whenever possible because they are picked up by all processes launched by `bin/hbase`. +# Use system properties when you launch multiple processes from the same configuration directory -- +# when you need to specify different configuration values for different hbase processes that are +# launched using the same HBase configuration (i.e., a single-host pseudo-distributed cluster or +# launching the `bin/hbase shell` from a host that is also running an instance of the master). See +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure +# for an inventory of configuration points and detailed explanations of each of them. +# +# Note also that as of this writing, the javaagent logs to stderr and is not configured along with +# the rest of HBase's logging configuration. +# +# `HBASE_OTEL_TRACING_ENABLED`, required. Enable attaching the opentelemetry javaagent to the +# process via support provided by `bin/hbase`. When this value us `false`, the agent is not added +# to the process launch arguments and all further OpenTelemetry configuration is ignored. +#export HBASE_OTEL_TRACING_ENABLED=true +# +# `OPENTELEMETRY_JAVAAGENT_PATH`, optional. Override the javaagent provided by HBase in `lib/trace` +# with an alternate. Use when you need to upgrade the agent version or swap out the official one +# for an alternative implementation. +#export OPENTELEMETRY_JAVAAGENT_PATH="" +# +# `OTEL_FOO_EXPORTER`, required. Specify an Exporter implementation per signal type. HBase only +# makes explicit use of the traces signal at this time, so the important one is +# `OTEL_TRACES_EXPORTER`. Specify its value based on the exporter required for your tracing +# environment. The other two should be uncommented and specified as `none`, otherwise the agent +# may report errors while attempting to export these other signals to an unconfigured destination. +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#exporters +#export OTEL_TRACES_EXPORTER="" +#export OTEL_METRICS_EXPORTER="none" +#export OTEL_LOGS_EXPORTER="none" +# +# `OTEL_SERVICE_NAME`, required. Specify "resource attributes", and specifically the `service.name`, +# as a unique value for each HBase process. OpenTelemetry allows for specifying this value in one +# of two ways, via environment variables with the `OTEL_` prefix, or via system properties with the +# `otel.` prefix. Which you use with HBase is decided based on whether this configuration file is +# read by a single process or shared by multiple HBase processes. For the default standalone mode +# or an environment where all processes share the same configuration file, use the `otel` system +# properties by uncommenting all of the `HBASE_FOO_OPTS` exports below. When this configuration file +# is being consumed by only a single process -- for example, from a systemd configuration or in a +# container template -- replace use of `HBASE_FOO_OPTS` with the standard `OTEL_SERVICE_NAME` and/or +# `OTEL_RESOURCE_ATTRIBUTES` environment variables. For further details, see +# https://github.com/open-telemetry/opentelemetry-java/tree/v1.15.0/sdk-extensions/autoconfigure#opentelemetry-resource +#export HBASE_CANARY_OPTS="${HBASE_CANARY_OPTS} -Dotel.resource.attributes=service.name=hbase-canary" +#export HBASE_HBCK_OPTS="${HBASE_HBCK_OPTS} -Dotel.resource.attributes=service.name=hbase-hbck" +#export HBASE_HBTOP_OPTS="${HBASE_HBTOP_OPTS} -Dotel.resource.attributes=service.name=hbase-hbtop" +#export HBASE_JSHELL_OPTS="${HBASE_JSHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-jshell" +#export HBASE_LTT_OPTS="${HBASE_LTT_OPTS} -Dotel.resource.attributes=service.name=hbase-loadtesttool" +#export HBASE_MASTER_OPTS="${HBASE_MASTER_OPTS} -Dotel.resource.attributes=service.name=hbase-master" +#export HBASE_PE_OPTS="${HBASE_PE_OPTS} -Dotel.resource.attributes=service.name=hbase-performanceevaluation" +#export HBASE_REGIONSERVER_OPTS="${HBASE_REGIONSERVER_OPTS} -Dotel.resource.attributes=service.name=hbase-regionserver" +#export HBASE_REST_OPTS="${HBASE_REST_OPTS} -Dotel.resource.attributes=service.name=hbase-rest" +#export HBASE_SHELL_OPTS="${HBASE_SHELL_OPTS} -Dotel.resource.attributes=service.name=hbase-shell" +#export HBASE_THRIFT_OPTS="${HBASE_THRIFT_OPTS} -Dotel.resource.attributes=service.name=hbase-thrift" +#export HBASE_ZOOKEEPER_OPTS="${HBASE_ZOOKEEPER_OPTS} -Dotel.resource.attributes=service.name=hbase-zookeeper" + +# +# JDK11+ JShell +# +# Additional arguments passed to jshell invocation +# export HBASE_JSHELL_ARGS="--startup DEFAULT --startup PRINTING --startup hbase_startup.jsh" diff --git a/hadoop-worker3/files/etc/hbase/conf/hbase-policy.xml b/hadoop-worker3/files/etc/hbase/conf/hbase-policy.xml new file mode 100644 index 0000000..5a0256d --- /dev/null +++ b/hadoop-worker3/files/etc/hbase/conf/hbase-policy.xml @@ -0,0 +1,53 @@ + + + + + + + security.client.protocol.acl + * + ACL for ClientProtocol and AdminProtocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.admin.protocol.acl + * + ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.masterregion.protocol.acl + * + ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + diff --git a/hadoop-worker3/files/etc/hbase/conf/hbase-site.xml b/hadoop-worker3/files/etc/hbase/conf/hbase-site.xml new file mode 100644 index 0000000..a8f52a1 --- /dev/null +++ b/hadoop-worker3/files/etc/hbase/conf/hbase-site.xml @@ -0,0 +1,70 @@ + + + + + + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://hadoop-master1.orb.local:8020/hbase + + + hbase.tmp.dir + hdfs://hadoop-master1.orb.local:8020/hbase/tmp + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.zookeeper.quorum + hadoop-master1.orb.local + + + hbase.zookeeper.property.clientPort + 2181 + + + hbase.master.info.port + 16010 + + diff --git a/hadoop-worker3/files/etc/hbase/conf/log4j2-hbtop.properties b/hadoop-worker3/files/etc/hbase/conf/log4j2-hbtop.properties new file mode 100644 index 0000000..de2f976 --- /dev/null +++ b/hadoop-worker3/files/etc/hbase/conf/log4j2-hbtop.properties @@ -0,0 +1,35 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# console +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %m%n + +rootLogger = WARN,console + +# ZooKeeper will still put stuff at WARN +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = ERROR + diff --git a/hadoop-worker3/files/etc/hbase/conf/log4j2.properties b/hadoop-worker3/files/etc/hbase/conf/log4j2.properties new file mode 100644 index 0000000..5ffcfda --- /dev/null +++ b/hadoop-worker3/files/etc/hbase/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +# logger.hbase.name = org.apache.hadoop.hbase +# logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/hadoop-worker3/files/etc/hbase/conf/regionservers b/hadoop-worker3/files/etc/hbase/conf/regionservers new file mode 100644 index 0000000..ee398e3 --- /dev/null +++ b/hadoop-worker3/files/etc/hbase/conf/regionservers @@ -0,0 +1,3 @@ +hadoop-worker1.orb.local +hadoop-worker2.orb.local +hadoop-worker3.orb.local \ No newline at end of file diff --git a/host_vars/local.yaml b/host_vars/local.yaml index b9a70d4..97d576d 100644 --- a/host_vars/local.yaml +++ b/host_vars/local.yaml @@ -47,6 +47,9 @@ ranger_version: 2.4.0 trino_include: true trino_version: 436 +hbase_include: true +hbase_version: 2.5.7 + iceberg_include: true iceberg_version: 1.4.2 diff --git a/templates/download.sh.j2 b/templates/download.sh.j2 index 6ddbdf3..3d294c6 100755 --- a/templates/download.sh.j2 +++ b/templates/download.sh.j2 @@ -59,6 +59,10 @@ download_if_not_exists ${MAVEN_MIRROR}/io/trino/trino-server/${TRINO_VERSION}/tr download_if_not_exists ${MAVEN_MIRROR}/io/trino/trino-cli/${TRINO_VERSION}/trino-cli-${TRINO_VERSION}-executable.jar {% endif %} +{% if hbase_include %} +download_if_not_exists ${APACHE_MIRROR}/hbase/${HBASE_VERSION}/hbase-${HBASE_VERSION}-bin.tar.gz +{% endif %} + MYSQL_JDBC_JAR_NAME=mysql-connector-j download_if_not_exists ${MAVEN_MIRROR}/com/mysql/${MYSQL_JDBC_JAR_NAME}/${MYSQL_JDBC_VERSION}/${MYSQL_JDBC_JAR_NAME}-${MYSQL_JDBC_VERSION}.jar diff --git a/templates/env.j2 b/templates/env.j2 index e09c90d..cdfbf3a 100644 --- a/templates/env.j2 +++ b/templates/env.j2 @@ -25,6 +25,7 @@ FLINK_VERSION={{ flink_version }} ZOOKEEPER_VERSION={{ zookeeper_version }} RANGER_VERSION={{ ranger_version }} TRINO_VERSION={{ trino_version }} +HBASE_VERSION={{ hbase_version }} GRAFANA_VERSION={{ grafana_version }} PROMETHEUS_VERSION=