This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new ae674a6 Optimize the bash script (#29)
ae674a6 is described below
commit ae674a61be10380c93e6a7f8518b475995fd0d62
Author: Junfan Zhang <[email protected]>
AuthorDate: Fri Jul 8 19:50:02 2022 +0800
Optimize the bash script (#29)
### What changes were proposed in this pull request?
Optimzie the bash script when deploying.
Main changelog
1. Use the `bin/hadoop` to get the hadoop deps.
2. Separate the log file for coordinator and server.
### Why are the changes needed?
To be compatible with the other hadoop distribution Including Apache
Hadoop, like CDH.
### Does this PR introduce _any_ user-facing change?
Yes.
### How was this patch tested?
Manual test.
---
bin/start-coordinator.sh | 23 ++++++++++++++---------
bin/start-shuffle-server.sh | 24 +++++++++++++++---------
conf/log4j.properties | 2 +-
3 files changed, 30 insertions(+), 19 deletions(-)
diff --git a/bin/start-coordinator.sh b/bin/start-coordinator.sh
index f900e24..581910d 100644
--- a/bin/start-coordinator.sh
+++ b/bin/start-coordinator.sh
@@ -32,8 +32,15 @@ cd $COORDINATOR_HOME
source "${COORDINATOR_HOME}/bin/rss-env.sh"
source "${COORDINATOR_HOME}/bin/utils.sh"
+if [ -z "$HADOOP_HOME" ]; then
+ echo "No env HADOOP_HOME."
+ exit 1
+fi
+
+export JAVA_HOME
+
HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
-HADOOP_DEPENDENCY=$HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*
+HADOOP_DEPENDENCY=`$HADOOP_HOME/bin/hadoop classpath --glob`
echo "Check process existence"
is_jvm_process_running $JPS $MAIN_CLASS
@@ -45,11 +52,6 @@ for file in $(ls ${JAR_DIR}/coordinator/*.jar 2>/dev/null);
do
CLASSPATH=$CLASSPATH:$file
done
-if [ -z "$HADOOP_HOME" ]; then
- echo "No env HADOOP_HOME."
- exit 1
-fi
-
if [ -z "$HADOOP_CONF_DIR" ]; then
echo "No env HADOOP_CONF_DIR."
exit 1
@@ -71,10 +73,13 @@ JVM_ARGS=" -server \
-XX:InitiatingHeapOccupancyPercent=45 "
ARGS=""
-if [ -f ./conf/log4j.properties ]; then
- ARGS="$ARGS -Dlog4j.configuration=file:./conf/log4j.properties"
+
+LOG_CONF_FILE="./conf/log4j.properties"
+LOG_PATH="./logs/coordinator.log"
+if [ -f ${LOG_CONF_FILE} ]; then
+ ARGS="$ARGS -Dlog4j.configuration=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}"
else
- echo "Exit with error: $conf/log4j.properties file doesn't exist."
+ echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
exit 1
fi
diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh
index 417b426..196960c 100644
--- a/bin/start-shuffle-server.sh
+++ b/bin/start-shuffle-server.sh
@@ -30,8 +30,15 @@ cd $SHUFFLE_SERVER_HOME
source "${SHUFFLE_SERVER_HOME}/bin/rss-env.sh"
source "${SHUFFLE_SERVER_HOME}/bin/utils.sh"
+if [ -z "$HADOOP_HOME" ]; then
+ echo "No env HADOOP_HOME."
+ exit 1
+fi
+
+export JAVA_HOME
+
HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
-HADOOP_DEPENDENCY=$HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*
+HADOOP_DEPENDENCY=`$HADOOP_HOME/bin/hadoop classpath --glob`
CONF_FILE="./conf/server.conf "
MAIN_CLASS="org.apache.uniffle.server.ShuffleServer"
@@ -46,11 +53,6 @@ for file in $(ls ${JAR_DIR}/server/*.jar 2>/dev/null); do
CLASSPATH=$CLASSPATH:$file
done
-if [ -z "$HADOOP_HOME" ]; then
- echo "No env HADOOP_HOME."
- exit 1
-fi
-
if [ -z "$HADOOP_CONF_DIR" ]; then
echo "No env HADOOP_CONF_DIR."
exit 1
@@ -85,10 +87,14 @@ JVM_ARGS=" -server \
-Xloggc:./logs/gc-%t.log"
ARGS=""
-if [ -f ./conf/log4j.properties ]; then
- ARGS="$ARGS -Dlog4j.configuration=file:./conf/log4j.properties"
+
+LOG_CONF_FILE="./conf/log4j.properties"
+LOG_PATH="./logs/shuffle_server.log"
+
+if [ -f ${LOG_CONF_FILE} ]; then
+ ARGS="$ARGS -Dlog4j.configuration=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}"
else
- echo "Exit with error: $conf/log4j.properties file doesn't exist."
+ echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
exit 1
fi
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 79a104b..47f2eef 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -22,7 +22,7 @@ log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p
%c{1}: %m%n
log4j.appender.RollingAppender=org.apache.log4j.RollingFileAppender
-log4j.appender.RollingAppender.File=./logs/rss.log
+log4j.appender.RollingAppender.File=${log.path}
log4j.appender.RollingAppender.MaxFileSize=50MB
log4j.appender.RollingAppender.MaxBackupIndex=10
log4j.appender.RollingAppender.layout=org.apache.log4j.PatternLayout
