This is an automated email from the ASF dual-hosted git repository.
xianjin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 13fc9840d [MINOR] fix(script): Fix log4j2.xml script and intellij
launch conf to support output audit logs (#2028)
13fc9840d is described below
commit 13fc9840d117552d4f9ce10b9dc5a0b88ae645be
Author: maobaolong <[email protected]>
AuthorDate: Mon Aug 19 19:29:59 2024 +0800
[MINOR] fix(script): Fix log4j2.xml script and intellij launch conf to
support output audit logs (#2028)
### What changes were proposed in this pull request?
Fix log4j2.xml script and intellij launch conf to support output audit logs
### Why are the changes needed?
Without this PR, there are some issues
- Extra empty logs file generated.
- The audit logs cannot generated in logs folder when launch Uniffle
servers from IntelliJ IDEA.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Start servers and check the logs.
```
ll incubator-uniffle/logs
total 44K
-rw-r--r-- 1 mbl staff 5.7K 8 9 17:36 coordinator.log
-rw-r--r-- 1 mbl staff 625 8 9 17:36 coordinator_rpc_audit.log
-rw-r--r-- 1 mbl staff 1.1K 8 9 17:34 dashboard.log
-rw-r--r-- 1 mbl staff 15K 8 9 17:36 shuffle_server.log
-rw-r--r-- 1 mbl staff 12K 8 9 17:36 shuffle_server_rpc_audit.log
```
---
bin/start-coordinator.sh | 2 +-
bin/start-shuffle-server.sh | 2 +-
conf/log4j2.xml | 19 +++++++------------
dev/intellij/install-runconfig.sh | 4 ++--
dev/intellij/runConfs/CoordinatorServer.run | 2 +-
dev/intellij/runConfs/ShuffleServer.run | 2 +-
dev/local_dev_template/log4j2.xml | 19 +++++++------------
7 files changed, 20 insertions(+), 30 deletions(-)
diff --git a/bin/start-coordinator.sh b/bin/start-coordinator.sh
index 2e790f8a0..55b9c5439 100755
--- a/bin/start-coordinator.sh
+++ b/bin/start-coordinator.sh
@@ -97,7 +97,7 @@ GC_LOG_ARGS_NEW=" -XX:+IgnoreUnrecognizedVMOptions \
JVM_LOG_ARGS=""
if [ -f ${LOG_CONF_FILE} ]; then
- JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}
-Dcoordinator.rpc.audit.log.path=${COORDINATOR_RPC_AUDIT_LOG_PATH}"
+ JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH} -Drpc.audit.log.path=${COORDINATOR_RPC_AUDIT_LOG_PATH}"
else
echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
exit 1
diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh
index bc52c65ee..b98c66799 100755
--- a/bin/start-shuffle-server.sh
+++ b/bin/start-shuffle-server.sh
@@ -135,7 +135,7 @@ GC_LOG_ARGS_NEW=" -XX:+IgnoreUnrecognizedVMOptions \
JVM_LOG_ARGS=""
if [ -f ${LOG_CONF_FILE} ]; then
- JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}
-Dshuffle.server.storage.audit.log.path=${SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH}
-Dshuffle.server.rpc.audit.log.path=${SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH}"
+ JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}
-Dstorage.audit.log.path=${SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH}
-Drpc.audit.log.path=${SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH}"
else
echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
exit 1
diff --git a/conf/log4j2.xml b/conf/log4j2.xml
index 3bbbb466b..027ad1d34 100644
--- a/conf/log4j2.xml
+++ b/conf/log4j2.xml
@@ -27,21 +27,16 @@
</Policies>
<DefaultRolloverStrategy max="10"/>
</RollingFile>
- <RollingFile name="shuffleServerStorageAuditAppender"
fileName="${sys:shuffle.server.storage.audit.log.path}"
filePattern="${sys:shuffle.server.storage.audit.log.path}.%i">
+ <RollingFile name="StorageAuditAppender"
fileName="${sys:storage.audit.log.path}"
filePattern="${sys:storage.audit.log.path}.%i"
+ createOnDemand="true">
<PatternLayout pattern="%m%n"/>
<Policies>
<SizeBasedTriggeringPolicy size="2GB"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
</RollingFile>
- <RollingFile name="shuffleServerRpcAuditAppender"
fileName="${sys:shuffle.server.rpc.audit.log.path}"
filePattern="${sys:shuffle.server.rpc.audit.log.path}.%i">
- <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
- <Policies>
- <SizeBasedTriggeringPolicy size="2GB"/>
- </Policies>
- <DefaultRolloverStrategy max="10"/>
- </RollingFile>
- <RollingFile name="coordinatorRpcAuditAppender"
fileName="${sys:coordinator.rpc.audit.log.path}"
filePattern="${sys:coordinator.rpc.audit.log.path}.%i">
+ <RollingFile name="RPCAuditAppender" fileName="${sys:rpc.audit.log.path}"
filePattern="${sys:rpc.audit.log.path}.%i"
+ createOnDemand="true">
<PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
<Policies>
<SizeBasedTriggeringPolicy size="2GB"/>
@@ -63,13 +58,13 @@
<AppenderRef ref="RollingAppender"/>
</Logger>
<Logger name="SHUFFLE_SERVER_STORAGE_AUDIT_LOG" level="INFO"
additivity="false">
- <AppenderRef ref="shuffleServerStorageAuditAppender"/>
+ <AppenderRef ref="StorageAuditAppender"/>
</Logger>
<Logger name="SHUFFLE_SERVER_RPC_AUDIT_LOG" level="INFO"
additivity="false">
- <AppenderRef ref="shuffleServerRpcAuditAppender"/>
+ <AppenderRef ref="RPCAuditAppender"/>
</Logger>
<Logger name="COORDINATOR_RPC_AUDIT_LOG" level="INFO" additivity="false">
- <AppenderRef ref="coordinatorRpcAuditAppender"/>
+ <AppenderRef ref="RPCAuditAppender"/>
</Logger>
</Loggers>
</Configuration>
diff --git a/dev/intellij/install-runconfig.sh
b/dev/intellij/install-runconfig.sh
index f290afa5c..20dbb709b 100755
--- a/dev/intellij/install-runconfig.sh
+++ b/dev/intellij/install-runconfig.sh
@@ -23,11 +23,11 @@ set -o errexit # exit the script if any statement returns
a non-true return va
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
SRC_DIR="$SCRIPT_DIR/runConfs"
-DEST_DIR="$SCRIPT_DIR/../../.idea/runConfigurations1/"
+DEST_DIR="$SCRIPT_DIR/../../.idea/runConfigurations/"
mkdir -p "$DEST_DIR"
ls -1 "$SRC_DIR" | xargs -n1 -I FILE cp "$SRC_DIR/FILE" "$DEST_DIR/FILE.xml"
SRC_DIR="$SCRIPT_DIR/../../dev/local_dev_template"
-DEST_DIR="$SCRIPT_DIR/../../.idea/local_dev1/"
+DEST_DIR="$SCRIPT_DIR/../../.idea/local_dev/"
mkdir -p "$DEST_DIR"
ls -1 "$SRC_DIR" | xargs -n1 -I FILE cp "$SRC_DIR/FILE" "$DEST_DIR"
diff --git a/dev/intellij/runConfs/CoordinatorServer.run
b/dev/intellij/runConfs/CoordinatorServer.run
index 69ec9de90..c9aee0660 100644
--- a/dev/intellij/runConfs/CoordinatorServer.run
+++ b/dev/intellij/runConfs/CoordinatorServer.run
@@ -20,7 +20,7 @@
<option name="MAIN_CLASS_NAME"
value="org.apache.uniffle.coordinator.CoordinatorServer" />
<module name="coordinator" />
<option name="PROGRAM_PARAMETERS" value="--conf
$PROJECT_DIR$/.idea/local_dev/coordinator.conf" />
- <option name="VM_PARAMETERS"
value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml
-Dlog.path=$PROJECT_DIR$/logs/coordinator.log" />
+ <option name="VM_PARAMETERS"
value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml
-Dlog.path=$PROJECT_DIR$/logs/coordinator.log
-Drpc.audit.log.path=$PROJECT_DIR$/logs/coordinator_rpc_audit.log" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="org.apache.uniffle.coordinator.*" />
diff --git a/dev/intellij/runConfs/ShuffleServer.run
b/dev/intellij/runConfs/ShuffleServer.run
index 0f29a1b05..c7eb11cad 100644
--- a/dev/intellij/runConfs/ShuffleServer.run
+++ b/dev/intellij/runConfs/ShuffleServer.run
@@ -20,7 +20,7 @@
<option name="MAIN_CLASS_NAME"
value="org.apache.uniffle.server.ShuffleServer" />
<module name="shuffle-server" />
<option name="PROGRAM_PARAMETERS" value="--conf
$PROJECT_DIR$/.idea/local_dev/server.conf" />
- <option name="VM_PARAMETERS"
value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml
-Dlog.path=$PROJECT_DIR$/logs/shuffle_server.log" />
+ <option name="VM_PARAMETERS"
value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/.idea/local_dev/log4j2.xml
-Dlog.path=$PROJECT_DIR$/logs/shuffle_server.log
-Drpc.audit.log.path=$PROJECT_DIR$/logs/shuffle_server_rpc_audit.log
-Dstorage.audit.log.path=$PROJECT_DIR$/logs/shuffle_server_storage_audit.log "
/>
<extension name="coverage">
<pattern>
<option name="PATTERN" value="org.apache.uniffle.server.*" />
diff --git a/dev/local_dev_template/log4j2.xml
b/dev/local_dev_template/log4j2.xml
index 95830d65d..4b5699d58 100644
--- a/dev/local_dev_template/log4j2.xml
+++ b/dev/local_dev_template/log4j2.xml
@@ -27,21 +27,16 @@
</Policies>
<DefaultRolloverStrategy max="10"/>
</RollingFile>
- <RollingFile name="shuffleServerStorageAuditAppender"
fileName="${sys:shuffle.server.storage.audit.log.path}"
filePattern="${sys:shuffle.server.storage.audit.log.path}.%i">
+ <RollingFile name="StorageAuditAppender"
fileName="${sys:storage.audit.log.path}"
filePattern="${sys:storage.audit.log.path}.%i"
+ createOnDemand="true">
<PatternLayout pattern="%m%n"/>
<Policies>
<SizeBasedTriggeringPolicy size="2GB"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
</RollingFile>
- <RollingFile name="shuffleServerRpcAuditAppender"
fileName="${sys:shuffle.server.rpc.audit.log.path}"
filePattern="${sys:shuffle.server.rpc.audit.log.path}.%i">
- <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
- <Policies>
- <SizeBasedTriggeringPolicy size="2GB"/>
- </Policies>
- <DefaultRolloverStrategy max="10"/>
- </RollingFile>
- <RollingFile name="coordinatorRpcAuditAppender"
fileName="${sys:coordinator.rpc.audit.log.path}"
filePattern="${sys:coordinator.rpc.audit.log.path}.%i">
+ <RollingFile name="RPCAuditAppender" fileName="${sys:rpc.audit.log.path}"
filePattern="${sys:rpc.audit.log.path}.%i"
+ createOnDemand="true">
<PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] %m%n"/>
<Policies>
<SizeBasedTriggeringPolicy size="2GB"/>
@@ -63,13 +58,13 @@
<AppenderRef ref="console"/>
</Logger>
<Logger name="SHUFFLE_SERVER_STORAGE_AUDIT_LOG" level="INFO"
additivity="false">
- <AppenderRef ref="shuffleServerStorageAuditAppender"/>
+ <AppenderRef ref="StorageAuditAppender"/>
</Logger>
<Logger name="SHUFFLE_SERVER_RPC_AUDIT_LOG" level="INFO"
additivity="false">
- <AppenderRef ref="shuffleServerRpcAuditAppender"/>
+ <AppenderRef ref="RPCAuditAppender"/>
</Logger>
<Logger name="COORDINATOR_RPC_AUDIT_LOG" level="INFO" additivity="false">
- <AppenderRef ref="coordinatorRpcAuditAppender"/>
+ <AppenderRef ref="RPCAuditAppender"/>
</Logger>
</Loggers>
</Configuration>
