Merge pull request #224 from FalkorDB/222-bug-aof-file-ended-up-with-…

…9gb-in-size

changed aof-rewrite

falkordb-cluster/Dockerfile

@@ -16,9 +16,13 @@ RUN cargo build --release
 
 FROM falkordb/falkordb:$FALKORDB_VERSION
 
-RUN apt-get update && apt-get install -y curl jq openssl
+RUN apt-get update && apt-get install -y curl jq openssl cron
+
+RUN useradd -u 1000 -m falkordb && usermod -aG crontab falkordb && \
+  touch /var/run/crond.pid && chown root:crontab /var/run/crond.pid \
+  && chmod 660 /var/run/crond.pid \
+  && chmod u+s /usr/sbin/cron
 
-RUN mkdir -p /falkordb
 
 WORKDIR /falkordb
 

falkordb-cluster/cluster-entrypoint.sh

@@ -37,6 +37,7 @@ FALKORDB_TIMEOUT_MAX=${FALKORDB_TIMEOUT_MAX:-0}
 FALKORDB_TIMEOUT_DEFAULT=${FALKORDB_TIMEOUT_DEFAULT:-0}
 FALKORDB_VKEY_MAX_ENTITY_COUNT=${FALKORDB_VKEY_MAX_ENTITY_COUNT:-4611686000000000000}
 MEMORY_LIMIT=${MEMORY_LIMIT:-''}
+AOF_CRON_EXPRESSION=${AOF_CRON_EXPRESSION:-'0 */12 * * *'}
 
 # If vars are <nil>, set it to 0
 if [[ "$FALKORDB_QUERY_MEM_CAPACITY" == "<nil>" ]]; then
@@ -76,6 +77,13 @@ if [[ $OMNISTRATE_ENVIRONMENT_TYPE != "PROD" ]];then
 fi
 
 
+rewrite_aof_cronjob(){
+  # This function runs the BGREWRITEAOF command every 12 hours to prevent the AOF file from growing too large.
+  # The command is run every 12 hours to prevent the AOF file from growing too large.
+  cron
+  crontab <<< "$AOF_CRON_EXPRESSION $(which redis-cli) $AUTH_CONNECTION_STRING $TLS_CONNECTION_STRING BGREWRITEAOF"
+}
+
 meet_unknown_nodes(){
   # Had to add sleep until things are stable (nodes that can communicate should be given time to do so)
   sleep 30
@@ -460,6 +468,8 @@ if [[ $RUN_METRICS -eq 1 ]]; then
   redis_exporter_pid=$!
 fi
 
+rewrite_aof_cronjob
+
 while true; do
   sleep 1
 done

falkordb-cluster/node.conf

@@ -47,7 +47,7 @@ cluster-node-timeout 5000
 cluster-announce-hostname $NODE_HOST
 masterauth $ADMIN_PASSWORD
 requirepass $ADMIN_PASSWORD
-auto-aof-rewrite-percentage 100
+auto-aof-rewrite-percentage 20
 auto-aof-rewrite-min-size 64mb
 
 

falkordb-node/Dockerfile

@@ -16,7 +16,13 @@ RUN cargo build --release
 
 FROM falkordb/falkordb:$FALKORDB_VERSION
 
-RUN apt-get update && apt-get install -y curl jq openssl
+RUN apt-get update && apt-get install -y curl jq openssl cron
+
+RUN useradd -u 1000 -m falkordb && usermod -aG crontab falkordb && \
+  touch /var/run/crond.pid && chown root:crontab /var/run/crond.pid \
+  && chmod 660 /var/run/crond.pid \
+  && chmod u+s /usr/sbin/cron
+
 
 RUN mkdir -p /falkordb
 

falkordb-node/node-entrypoint.sh

@@ -74,6 +74,7 @@ DEBUG=${DEBUG:-0}
 REPLACE_NODE_CONF=${REPLACE_NODE_CONF:-0}
 REPLACE_SENTINEL_CONF=${REPLACE_SENTINEL_CONF:-0}
 TLS_CONNECTION_STRING=$(if [[ $TLS == "true" ]]; then echo "--tls --cacert $ROOT_CA_PATH"; else echo ""; fi)
+AUTH_CONNECTION_STRING="-a $ADMIN_PASSWORD --no-auth-warning"
 SAVE_LOGS_TO_FILE=${SAVE_LOGS_TO_FILE:-1}
 LOG_LEVEL=${LOG_LEVEL:-notice}
 
@@ -83,11 +84,22 @@ FALKORDB_LOG_FILE_PATH=$(if [[ $SAVE_LOGS_TO_FILE -eq 1 ]]; then echo $DATA_DIR/
 SENTINEL_LOG_FILE_PATH=$(if [[ $SAVE_LOGS_TO_FILE -eq 1 ]]; then echo $DATA_DIR/sentinel_$DATE_NOW.log; else echo ""; fi)
 NODE_CONF_FILE=$DATA_DIR/node.conf
 SENTINEL_CONF_FILE=$DATA_DIR/sentinel.conf
+AOF_CRON_EXPRESSION=${AOF_CRON_EXPRESSION:-'0 */12 * * *'}
+
+
 
 if [[ $OMNISTRATE_ENVIRONMENT_TYPE != "PROD" ]]; then
   DEBUG=1
 fi
 
+rewrite_aof_cronjob(){
+  # This function runs the BGREWRITEAOF command every 12 hours to prevent the AOF file from growing too large.
+  # The command is run every 12 hours to prevent the AOF file from growing too large.
+  cron
+  crontab <<< "$AOF_CRON_EXPRESSION $(which redis-cli) $AUTH_CONNECTION_STRING $TLS_CONNECTION_STRING BGREWRITEAOF"
+}
+
+
 dump_conf_files() {
   echo "Dumping configuration files"
 
@@ -545,6 +557,11 @@ if [[ $RUN_METRICS -eq 1 ]]; then
   redis_exporter_pid=$!
 fi
 
+if [[ ! "$NODE_NAME" =~ sentinel.* ]]; then
+  rewrite_aof_cronjob
+fi
+
+
 if [[ $DEBUG -eq 1 && $RUN_SENTINEL -eq 1 ]] && [[ "$NODE_INDEX" == "1" || "$NODE_INDEX" == "0" ]]; then
   # Check for crossed namespace
   echo "Checking for crossed namespace"

falkordb-node/node.conf

@@ -6,7 +6,7 @@ replica-announce-ip $NODE_HOST
 replica-announce-port $NODE_PORT
 masterauth $ADMIN_PASSWORD
 requirepass $ADMIN_PASSWORD
-auto-aof-rewrite-percentage 100
+auto-aof-rewrite-percentage 20
 auto-aof-rewrite-min-size 64mb
 
 # Injected configs