This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new b924acacb [MINOR] feat(build): Allow to build distribution without
some modules (#1525)
b924acacb is described below
commit b924acacb0c555370a593f3a069187cf8b8081d7
Author: Enrico Minack <[email protected]>
AuthorDate: Sat Feb 17 03:28:35 2024 +0100
[MINOR] feat(build): Allow to build distribution without some modules
(#1525)
### What changes were proposed in this pull request?
Allow to not build some modules when running `./build_distribution.sh`:
./build_distribution.sh --without-mr --without-tez --without-spark
--without-dashboard
### Why are the changes needed?
Some modules may not be needed and especially `dashboard` takes very long
to build.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manually.
---
build_distribution.sh | 139 +++++++++++++++++++++++++++++++++-----------------
1 file changed, 92 insertions(+), 47 deletions(-)
diff --git a/build_distribution.sh b/build_distribution.sh
index fe9ce2148..1483be773 100755
--- a/build_distribution.sh
+++ b/build_distribution.sh
@@ -37,18 +37,25 @@ function exit_with_usage() {
echo
"+------------------------------------------------------------------------------------------------------+"
echo "| ./build_distribution.sh [--spark2-profile <spark2 profile id>]
[--spark2-mvn <custom maven options>] |"
echo "| [--spark3-profile <spark3 profile id>]
[--spark3-mvn <custom maven options>] |"
- echo "| [--hadoop-profile <hadoop profile id>]
|"
+ echo "| [--hadoop-profile <hadoop profile id>]
[--without-mr] [--without-tez] |"
+ echo "| [--without-spark] [--without-spark2]
[--without-spark3] [--without-dashboard]|"
+ echo "| [--name <custom name>]
|"
echo "| <maven build options>
|"
echo
"+------------------------------------------------------------------------------------------------------+"
exit 1
}
-MVN_OPTS="-Pdashboard"
SPARK2_PROFILE_ID="spark2"
SPARK2_MVN_OPTS=""
SPARK3_PROFILE_ID="spark3"
SPARK3_MVN_OPTS=""
HADOOP_PROFILE_ID="hadoop2.8"
+WITH_MR="true"
+WITH_TEZ="true"
+WITH_SPARK="true"
+WITH_SPARK2="true"
+WITH_SPARK3="true"
+WITH_DASHBOARD="true"
NAME=none
while (( "$#" )); do
case $1 in
@@ -72,6 +79,25 @@ while (( "$#" )); do
HADOOP_PROFILE_ID=$2
shift
;;
+ --without-mr)
+ WITH_MR="false"
+ ;;
+ --without-tez)
+ WITH_TEZ="false"
+ ;;
+ --without-spark)
+ WITH_SPARK2="false"
+ WITH_SPARK3="false"
+ ;;
+ --without-spark2)
+ WITH_SPARK2="false"
+ ;;
+ --without-spark3)
+ WITH_SPARK3="false"
+ ;;
+ --without-dashboard)
+ WITH_DASHBOARD="false"
+ ;;
--name)
NAME="$2"
shift
@@ -139,7 +165,7 @@ export MAVEN_OPTS="${MAVEN_OPTS:--Xmx2g
-XX:ReservedCodeCacheSize=1g}"
# Store the command as an array because $MVN variable might have spaces in it.
# Normal quoting tricks don't work.
# See: http://mywiki.wooledge.org/BashFAQ/050
-BUILD_COMMAND=("$MVN" clean package -DskipTests $MVN_OPTS $@)
+BUILD_COMMAND=("$MVN" clean package -DskipTests $@)
# Actually build the jar
echo -e "\nBuilding with..."
@@ -170,13 +196,6 @@ echo "copy $COORDINATOR_JAR to ${COORDINATOR_JAR_DIR}"
cp $COORDINATOR_JAR ${COORDINATOR_JAR_DIR}
cp "${RSS_HOME}"/coordinator/target/jars/* ${COORDINATOR_JAR_DIR}
-DASHBOARD_JAR_DIR="${DISTDIR}/jars/dashboard"
-mkdir -p $DASHBOARD_JAR_DIR
-DASHBOARD_JAR="${RSS_HOME}/dashboard/target/dashboard-${VERSION}.jar"
-echo "copy $DASHBOARD_JAR to ${DASHBOARD_JAR_DIR}"
-cp $DASHBOARD_JAR $DASHBOARD_JAR_DIR
-cp "${RSS_HOME}"/dashboard/target/jars/* ${DASHBOARD_JAR_DIR}
-
CLI_JAR_DIR="${DISTDIR}/jars/cli"
mkdir -p $CLI_JAR_DIR
CLI_JAR="${RSS_HOME}/cli/target/cli-${VERSION}.jar"
@@ -187,52 +206,78 @@ cp "${RSS_HOME}"/cli/target/jars/* ${CLI_JAR_DIR}
CLIENT_JAR_DIR="${DISTDIR}/jars/client"
mkdir -p $CLIENT_JAR_DIR
-BUILD_COMMAND_SPARK2=("$MVN" clean package -P$SPARK2_PROFILE_ID -pl
client-spark/spark2-shaded -DskipTests -am $@ $SPARK2_MVN_OPTS)
# Actually build the jar
-echo -e "\nBuilding with..."
-echo -e "\$ ${BUILD_COMMAND_SPARK2[@]}\n"
+if [ "$WITH_SPARK2" == "true" ]; then
+ BUILD_COMMAND_SPARK2=("$MVN" clean package -P$SPARK2_PROFILE_ID -pl
client-spark/spark2-shaded -DskipTests -am $@ $SPARK2_MVN_OPTS)
+
+ echo -e "\nBuilding with..."
+ echo -e "\$ ${BUILD_COMMAND_SPARK2[@]}\n"
+ "${BUILD_COMMAND_SPARK2[@]}"
+
+ SPARK_CLIENT2_JAR_DIR="${CLIENT_JAR_DIR}/spark2"
+ mkdir -p $SPARK_CLIENT2_JAR_DIR
+
SPARK_CLIENT2_JAR="${RSS_HOME}/client-spark/spark2-shaded/target/rss-client-spark2-shaded-${VERSION}.jar"
+ echo "copy $SPARK_CLIENT2_JAR to ${SPARK_CLIENT2_JAR_DIR}"
+ cp $SPARK_CLIENT2_JAR ${SPARK_CLIENT2_JAR_DIR}
+fi
-"${BUILD_COMMAND_SPARK2[@]}"
+if [ "$WITH_SPARK3" == "true" ]; then
+ BUILD_COMMAND_SPARK3=("$MVN" clean package -P$SPARK3_PROFILE_ID -pl
client-spark/spark3-shaded -DskipTests -am $@ $SPARK3_MVN_OPTS)
-SPARK_CLIENT2_JAR_DIR="${CLIENT_JAR_DIR}/spark2"
-mkdir -p $SPARK_CLIENT2_JAR_DIR
+ echo -e "\nBuilding with..."
+ echo -e "\$ ${BUILD_COMMAND_SPARK3[@]}\n"
+ "${BUILD_COMMAND_SPARK3[@]}"
-SPARK_CLIENT2_JAR="${RSS_HOME}/client-spark/spark2-shaded/target/rss-client-spark2-shaded-${VERSION}.jar"
-echo "copy $SPARK_CLIENT2_JAR to ${SPARK_CLIENT2_JAR_DIR}"
-cp $SPARK_CLIENT2_JAR ${SPARK_CLIENT2_JAR_DIR}
+ SPARK_CLIENT3_JAR_DIR="${CLIENT_JAR_DIR}/spark3"
+ mkdir -p $SPARK_CLIENT3_JAR_DIR
+
SPARK_CLIENT3_JAR="${RSS_HOME}/client-spark/spark3-shaded/target/rss-client-spark3-shaded-${VERSION}.jar"
+ echo "copy $SPARK_CLIENT3_JAR to ${SPARK_CLIENT3_JAR_DIR}"
+ cp $SPARK_CLIENT3_JAR $SPARK_CLIENT3_JAR_DIR
+fi
-BUILD_COMMAND_SPARK3=("$MVN" clean package -P$SPARK3_PROFILE_ID -pl
client-spark/spark3-shaded -DskipTests -am $@ $SPARK3_MVN_OPTS)
+if [ "$WITH_MR" == "true" ]; then
+ BUILD_COMMAND_MR=("$MVN" clean package -Pmr,$HADOOP_PROFILE_ID -pl
client-mr/core -DskipTests -am $@)
-echo -e "\nBuilding with..."
-echo -e "\$ ${BUILD_COMMAND_SPARK3[@]}\n"
-"${BUILD_COMMAND_SPARK3[@]}"
+ echo -e "\nBuilding with..."
+ echo -e "\$ ${BUILD_COMMAND_MR[@]}\n"
+ "${BUILD_COMMAND_MR[@]}"
-SPARK_CLIENT3_JAR_DIR="${CLIENT_JAR_DIR}/spark3"
-mkdir -p $SPARK_CLIENT3_JAR_DIR
-SPARK_CLIENT3_JAR="${RSS_HOME}/client-spark/spark3-shaded/target/rss-client-spark3-shaded-${VERSION}.jar"
-echo "copy $SPARK_CLIENT3_JAR to ${SPARK_CLIENT3_JAR_DIR}"
-cp $SPARK_CLIENT3_JAR $SPARK_CLIENT3_JAR_DIR
+ MR_CLIENT_JAR_DIR="${CLIENT_JAR_DIR}/mr"
+ mkdir -p $MR_CLIENT_JAR_DIR
+
MR_CLIENT_JAR="${RSS_HOME}/client-mr/core/target/shaded/rss-client-mr-${VERSION}-shaded.jar"
+ echo "copy $MR_CLIENT_JAR to ${MR_CLIENT_JAR_DIR}"
+ cp $MR_CLIENT_JAR $MR_CLIENT_JAR_DIR
+fi
-BUILD_COMMAND_MR=("$MVN" clean package -Pmr,$HADOOP_PROFILE_ID -pl
client-mr/core -DskipTests -am $@)
-echo -e "\nBuilding with..."
-echo -e "\$ ${BUILD_COMMAND_MR[@]}\n"
-"${BUILD_COMMAND_MR[@]}"
-MR_CLIENT_JAR_DIR="${CLIENT_JAR_DIR}/mr"
-mkdir -p $MR_CLIENT_JAR_DIR
-MR_CLIENT_JAR="${RSS_HOME}/client-mr/core/target/shaded/rss-client-mr-${VERSION}-shaded.jar"
-echo "copy $MR_CLIENT_JAR to ${MR_CLIENT_JAR_DIR}"
-cp $MR_CLIENT_JAR $MR_CLIENT_JAR_DIR
-
-BUILD_COMMAND_TEZ=("$MVN" clean package -Ptez,$HADOOP_PROFILE_ID -pl
client-tez -DskipTests -am $@)
-echo -e "\nBuilding with..."
-echo -e "\$ ${BUILD_COMMAND_TEZ[@]}\n"
-"${BUILD_COMMAND_TEZ[@]}"
-TEZ_CLIENT_JAR_DIR="${CLIENT_JAR_DIR}/tez"
-mkdir -p $TEZ_CLIENT_JAR_DIR
-TEZ_CLIENT_JAR="${RSS_HOME}/client-tez/target/shaded/rss-client-tez-${VERSION}-shaded.jar"
-echo "copy $TEZ_CLIENT_JAR to ${TEZ_CLIENT_JAR_DIR}"
-cp $TEZ_CLIENT_JAR $TEZ_CLIENT_JAR_DIR
+if [ "$WITH_TEZ" == "true" ]; then
+ BUILD_COMMAND_TEZ=("$MVN" clean package -Ptez,$HADOOP_PROFILE_ID -pl
client-tez -DskipTests -am $@)
+
+ echo -e "\nBuilding with..."
+ echo -e "\$ ${BUILD_COMMAND_TEZ[@]}\n"
+ "${BUILD_COMMAND_TEZ[@]}"
+
+ TEZ_CLIENT_JAR_DIR="${CLIENT_JAR_DIR}/tez"
+ mkdir -p $TEZ_CLIENT_JAR_DIR
+
TEZ_CLIENT_JAR="${RSS_HOME}/client-tez/target/shaded/rss-client-tez-${VERSION}-shaded.jar"
+ echo "copy $TEZ_CLIENT_JAR to ${TEZ_CLIENT_JAR_DIR}"
+ cp $TEZ_CLIENT_JAR $TEZ_CLIENT_JAR_DIR
+fi
+
+if [ "$WITH_DASHBOARD" == "true" ]; then
+ BUILD_COMMAND_DASHBOARD=("$MVN" clean package -Pdashboard -pl dashboard
-DskipTests -am $@)
+
+ echo -e "\nBuilding with..."
+ echo -e "\$ ${BUILD_COMMAND_DASHBOARD[@]}\n"
+ "${BUILD_COMMAND_DASHBOARD[@]}"
+
+ DASHBOARD_JAR_DIR="${DISTDIR}/jars/dashboard"
+ mkdir -p $DASHBOARD_JAR_DIR
+ DASHBOARD_JAR="${RSS_HOME}/dashboard/target/dashboard-${VERSION}.jar"
+ echo "copy $DASHBOARD_JAR to ${DASHBOARD_JAR_DIR}"
+ cp $DASHBOARD_JAR $DASHBOARD_JAR_DIR
+ cp "${RSS_HOME}"/dashboard/target/jars/* ${DASHBOARD_JAR_DIR}
+fi
cp -r bin $DISTDIR
cp -r conf $DISTDIR
