[MINOR] fix(script): Fix log4j2.xml script and intellij launch conf t…

…o support output audit logs (apache#2028) ### What changes were proposed in this pull request? Fix log4j2.xml script and intellij launch conf to support output audit logs ### Why are the changes needed? Without this PR, there are some issues - Extra empty logs file generated. - The audit logs cannot generated in logs folder when launch Uniffle servers from IntelliJ IDEA. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Start servers and check the logs. ``` ll incubator-uniffle/logs total 44K -rw-r--r-- 1 mbl staff 5.7K 8 9 17:36 coordinator.log -rw-r--r-- 1 mbl staff 625 8 9 17:36 coordinator_rpc_audit.log -rw-r--r-- 1 mbl staff 1.1K 8 9 17:34 dashboard.log -rw-r--r-- 1 mbl staff 15K 8 9 17:36 shuffle_server.log -rw-r--r-- 1 mbl staff 12K 8 9 17:36 shuffle_server_rpc_audit.log ```
maobaolong · Aug 19, 2024 · 13fc984 · 13fc984
1 parent ba2302c
commit 13fc984
Show file tree

Hide file tree

Showing 7 changed files with 20 additions and 30 deletions.
diff --git a/bin/start-coordinator.sh b/bin/start-coordinator.sh
@@ -97,7 +97,7 @@ GC_LOG_ARGS_NEW=" -XX:+IgnoreUnrecognizedVMOptions \
 JVM_LOG_ARGS=""
 
 if [ -f ${LOG_CONF_FILE} ]; then
-  JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH} -Dcoordinator.rpc.audit.log.path=${COORDINATOR_RPC_AUDIT_LOG_PATH}"
+  JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH} -Drpc.audit.log.path=${COORDINATOR_RPC_AUDIT_LOG_PATH}"
 else
   echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
   exit 1

diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh
@@ -135,7 +135,7 @@ GC_LOG_ARGS_NEW=" -XX:+IgnoreUnrecognizedVMOptions \
 JVM_LOG_ARGS=""
 
 if [ -f ${LOG_CONF_FILE} ]; then
-  JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH} -Dshuffle.server.storage.audit.log.path=${SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH} -Dshuffle.server.rpc.audit.log.path=${SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH}"
+  JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH} -Dstorage.audit.log.path=${SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH} -Drpc.audit.log.path=${SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH}"
 else
   echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
   exit 1

diff --git a/conf/log4j2.xml b/conf/log4j2.xml
@@ -27,21 +27,16 @@
       </Policies>
       <DefaultRolloverStrategy max="10"/>
     </RollingFile>
-    <RollingFile name="shuffleServerStorageAuditAppender" fileName="${sys:shuffle.server.storage.audit.log.path}" filePattern="${sys:shuffle.server.storage.audit.log.path}.%i">
+    <RollingFile name="StorageAuditAppender" fileName="${sys:storage.audit.log.path}" filePattern="${sys:storage.audit.log.path}.%i"
+                 createOnDemand="true">
       <PatternLayout pattern="%m%n"/>
       <Policies>
         <SizeBasedTriggeringPolicy size="2GB"/>
       </Policies>
       <DefaultRolloverStrategy max="10"/>
     </RollingFile>
-    <RollingFile name="shuffleServerRpcAuditAppender" fileName="${sys:shuffle.server.rpc.audit.log.path}" filePattern="${sys:shuffle.server.rpc.audit.log.path}.%i">
-      <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
-      <Policies>
-        <SizeBasedTriggeringPolicy size="2GB"/>
-      </Policies>
-      <DefaultRolloverStrategy max="10"/>
-    </RollingFile>
-    <RollingFile name="coordinatorRpcAuditAppender" fileName="${sys:coordinator.rpc.audit.log.path}" filePattern="${sys:coordinator.rpc.audit.log.path}.%i">
+    <RollingFile name="RPCAuditAppender" fileName="${sys:rpc.audit.log.path}" filePattern="${sys:rpc.audit.log.path}.%i"
+                 createOnDemand="true">
       <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
       <Policies>
         <SizeBasedTriggeringPolicy size="2GB"/>
@@ -63,13 +58,13 @@
       <AppenderRef ref="RollingAppender"/>
     </Logger>
     <Logger name="SHUFFLE_SERVER_STORAGE_AUDIT_LOG" level="INFO" additivity="false">
-      <AppenderRef ref="shuffleServerStorageAuditAppender"/>
+      <AppenderRef ref="StorageAuditAppender"/>
     </Logger>
     <Logger name="SHUFFLE_SERVER_RPC_AUDIT_LOG" level="INFO" additivity="false">
-      <AppenderRef ref="shuffleServerRpcAuditAppender"/>
+      <AppenderRef ref="RPCAuditAppender"/>
     </Logger>
     <Logger name="COORDINATOR_RPC_AUDIT_LOG" level="INFO" additivity="false">
-      <AppenderRef ref="coordinatorRpcAuditAppender"/>
+      <AppenderRef ref="RPCAuditAppender"/>
     </Logger>
   </Loggers>
 </Configuration>
diff --git a/dev/intellij/install-runconfig.sh b/dev/intellij/install-runconfig.sh
@@ -23,11 +23,11 @@ set -o errexit   # exit the script if any statement returns a non-true return va
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 SRC_DIR="$SCRIPT_DIR/runConfs"
-DEST_DIR="$SCRIPT_DIR/../../.idea/runConfigurations1/"
+DEST_DIR="$SCRIPT_DIR/../../.idea/runConfigurations/"
 mkdir -p "$DEST_DIR"
 ls -1 "$SRC_DIR" | xargs -n1 -I FILE cp "$SRC_DIR/FILE" "$DEST_DIR/FILE.xml"
 
 SRC_DIR="$SCRIPT_DIR/../../dev/local_dev_template"
-DEST_DIR="$SCRIPT_DIR/../../.idea/local_dev1/"
+DEST_DIR="$SCRIPT_DIR/../../.idea/local_dev/"
 mkdir -p "$DEST_DIR"
 ls -1 "$SRC_DIR" | xargs -n1 -I FILE cp "$SRC_DIR/FILE" "$DEST_DIR"
diff --git a/dev/intellij/runConfs/CoordinatorServer.run b/dev/intellij/runConfs/CoordinatorServer.run
@@ -20,7 +20,7 @@
     <option name="MAIN_CLASS_NAME" value="org.apache.uniffle.coordinator.CoordinatorServer" />
     <module name="coordinator" />
     <option name="PROGRAM_PARAMETERS" value="--conf $PROJECT_DIR$/.idea/local_dev/coordinator.conf" />
-    <option name="VM_PARAMETERS" value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml -Dlog.path=$PROJECT_DIR$/logs/coordinator.log" />
+    <option name="VM_PARAMETERS" value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml -Dlog.path=$PROJECT_DIR$/logs/coordinator.log -Drpc.audit.log.path=$PROJECT_DIR$/logs/coordinator_rpc_audit.log" />
     <extension name="coverage">
       <pattern>
         <option name="PATTERN" value="org.apache.uniffle.coordinator.*" />

diff --git a/dev/intellij/runConfs/ShuffleServer.run b/dev/intellij/runConfs/ShuffleServer.run
@@ -20,7 +20,7 @@
     <option name="MAIN_CLASS_NAME" value="org.apache.uniffle.server.ShuffleServer" />
     <module name="shuffle-server" />
     <option name="PROGRAM_PARAMETERS" value="--conf $PROJECT_DIR$/.idea/local_dev/server.conf" />
-    <option name="VM_PARAMETERS" value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml -Dlog.path=$PROJECT_DIR$/logs/shuffle_server.log" />
+    <option name="VM_PARAMETERS" value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml -Dlog.path=$PROJECT_DIR$/logs/shuffle_server.log -Drpc.audit.log.path=$PROJECT_DIR$/logs/shuffle_server_rpc_audit.log -Dstorage.audit.log.path=$PROJECT_DIR$/logs/shuffle_server_storage_audit.log " />
     <extension name="coverage">
       <pattern>
         <option name="PATTERN" value="org.apache.uniffle.server.*" />

diff --git a/dev/local_dev_template/log4j2.xml b/dev/local_dev_template/log4j2.xml
@@ -27,21 +27,16 @@
       </Policies>
       <DefaultRolloverStrategy max="10"/>
     </RollingFile>
-    <RollingFile name="shuffleServerStorageAuditAppender" fileName="${sys:shuffle.server.storage.audit.log.path}" filePattern="${sys:shuffle.server.storage.audit.log.path}.%i">
+    <RollingFile name="StorageAuditAppender" fileName="${sys:storage.audit.log.path}" filePattern="${sys:storage.audit.log.path}.%i"
+                 createOnDemand="true">
       <PatternLayout pattern="%m%n"/>
       <Policies>
         <SizeBasedTriggeringPolicy size="2GB"/>
       </Policies>
       <DefaultRolloverStrategy max="10"/>
     </RollingFile>
-    <RollingFile name="shuffleServerRpcAuditAppender" fileName="${sys:shuffle.server.rpc.audit.log.path}" filePattern="${sys:shuffle.server.rpc.audit.log.path}.%i">
-      <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
-      <Policies>
-        <SizeBasedTriggeringPolicy size="2GB"/>
-      </Policies>
-      <DefaultRolloverStrategy max="10"/>
-    </RollingFile>
-    <RollingFile name="coordinatorRpcAuditAppender" fileName="${sys:coordinator.rpc.audit.log.path}" filePattern="${sys:coordinator.rpc.audit.log.path}.%i">
+    <RollingFile name="RPCAuditAppender" fileName="${sys:rpc.audit.log.path}" filePattern="${sys:rpc.audit.log.path}.%i"
+                 createOnDemand="true">
       <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
       <Policies>
         <SizeBasedTriggeringPolicy size="2GB"/>
@@ -63,13 +58,13 @@
       <AppenderRef ref="console"/>
     </Logger>
     <Logger name="SHUFFLE_SERVER_STORAGE_AUDIT_LOG" level="INFO" additivity="false">
-      <AppenderRef ref="shuffleServerStorageAuditAppender"/>
+      <AppenderRef ref="StorageAuditAppender"/>
     </Logger>
     <Logger name="SHUFFLE_SERVER_RPC_AUDIT_LOG" level="INFO" additivity="false">
-      <AppenderRef ref="shuffleServerRpcAuditAppender"/>
+      <AppenderRef ref="RPCAuditAppender"/>
     </Logger>
     <Logger name="COORDINATOR_RPC_AUDIT_LOG" level="INFO" additivity="false">
-      <AppenderRef ref="coordinatorRpcAuditAppender"/>
+      <AppenderRef ref="RPCAuditAppender"/>
     </Logger>
   </Loggers>
 </Configuration>