This is an automated email from the ASF dual-hosted git repository. jshao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 392c88129f2706043ebb87cc89e9e2cde5733647 Author: cxzl25 <cxz...@users.noreply.github.com> AuthorDate: Tue Jun 28 10:09:01 2022 +0800 Support build_distribution.sh to specify different mvn build options for Spark2 and Spark3 (#203) What changes were proposed in this pull request? Add --spark2-mvn, --spark3-mvn parameters in build_distribution.sh to support compiling different profiles, we can pass in different maven parameters, such as profile, spark version. Add --help parameters in build_distribution.sh, fix typo. gitignore ignores the tar package generated by build. README added how to use build_distribution.sh. Why are the changes needed? If we use such a command to build, Spark2 will also use the Spark3 version to compile, so we'd better distinguish the build options of different versions. ./build_distribution.sh -Pspark3.2 Does this PR introduce any user-facing change? No How was this patch tested? local test --- .gitignore | 1 + README.md | 16 ++++++++++++++++ build_distribution.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++++---- pom.xml | 4 ++-- 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 5c39d59..b6164b2 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ reports/ metastore_db/ derby.log dependency-reduced-pom.xml +rss-*.tgz diff --git a/README.md b/README.md index 9ad8299..51a1ed0 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,26 @@ To build it, run: mvn -DskipTests clean package +Build against profile Spark2(2.4.6) + + mvn -DskipTests clean package -Pspark2 + +Build against profile Spark3(3.1.2) + + mvn -DskipTests clean package -Pspark3 + +Build against Spark 3.2.x + + mvn -DskipTests clean package -Pspark3.2 + To package the Firestorm, run: ./build_distribution.sh +Package against Spark 3.2.x, run: + + ./build_distribution.sh --spark3-profile 'spark3.2' + rss-xxx.tgz will be generated for deployment ## Deploy diff --git a/build_distribution.sh b/build_distribution.sh index baf50e4..214a2ed 100755 --- a/build_distribution.sh +++ b/build_distribution.sh @@ -32,12 +32,57 @@ RSS_HOME="$( function exit_with_usage() { set +x - echo "$0 - tool for making binary distributions of Rmote Shuffle Service" + echo "./build_distribution.sh - Tool for making binary distributions of Remote Shuffle Service" echo "" - echo "usage:" + echo "Usage:" + echo "+------------------------------------------------------------------------------------------------------+" + echo "| ./build_distribution.sh [--spark2-profile <spark2 profile id>] [--spark2-mvn <custom maven options>] |" + echo "| [--spark3-profile <spark3 profile id>] [--spark3-mvn <custom maven options>] |" + echo "| <maven build options> |" + echo "+------------------------------------------------------------------------------------------------------+" exit 1 } +SPARK2_PROFILE_ID="spark2" +SPARK2_MVN_OPTS="" +SPARK3_PROFILE_ID="spark3" +SPARK3_MVN_OPTS="" +while (( "$#" )); do + case $1 in + --spark2-profile) + SPARK2_PROFILE_ID="$2" + shift + ;; + --spark2-mvn) + SPARK2_MVN_OPTS=$2 + shift + ;; + --spark3-profile) + SPARK3_PROFILE_ID="$2" + shift + ;; + --spark3-mvn) + SPARK3_MVN_OPTS=$2 + shift + ;; + --help) + exit_with_usage + ;; + --*) + echo "Error: $1 is not supported" + exit_with_usage + ;; + -*) + break + ;; + *) + echo "Error: $1 is not supported" + exit_with_usage + ;; + esac + shift +done + cd $RSS_HOME if [ -z "$JAVA_HOME" ]; then @@ -99,7 +144,7 @@ cp "${RSS_HOME}"/coordinator/target/jars/* ${COORDINATOR_JAR_DIR} CLIENT_JAR_DIR="${DISTDIR}/jars/client" mkdir -p $CLIENT_JAR_DIR -BUILD_COMMAND_SPARK2=("$MVN" clean package -Pspark2 -pl client-spark/spark2 -DskipTests -am $@) +BUILD_COMMAND_SPARK2=("$MVN" clean package -P$SPARK2_PROFILE_ID -pl client-spark/spark2 -DskipTests -am $@ $SPARK2_MVN_OPTS) # Actually build the jar echo -e "\nBuilding with..." @@ -114,7 +159,7 @@ SPARK_CLIENT2_JAR="${RSS_HOME}/client-spark/spark2/target/shaded/rss-client-spar echo "copy $SPARK_CLIENT2_JAR to ${SPARK_CLIENT2_JAR_DIR}" cp $SPARK_CLIENT2_JAR ${SPARK_CLIENT2_JAR_DIR} -BUILD_COMMAND_SPARK3=("$MVN" clean package -Pspark3 -pl client-spark/spark3 -DskipTests -am $@) +BUILD_COMMAND_SPARK3=("$MVN" clean package -P$SPARK3_PROFILE_ID -pl client-spark/spark3 -DskipTests -am $@ $SPARK3_MVN_OPTS) echo -e "\nBuilding with..." echo -e "\$ ${BUILD_COMMAND_SPARK3[@]}\n" diff --git a/pom.xml b/pom.xml index aef4fec..2f412f6 100644 --- a/pom.xml +++ b/pom.xml @@ -795,12 +795,12 @@ <plugin> <groupId>com.github.spotbugs</groupId> <artifactId>spotbugs-maven-plugin</artifactId> - <version>${spotbugs-maven-plugin.version}</version> + <version>${spotbugs-maven-plugin.version}</version> <dependencies> <dependency> <groupId>com.github.spotbugs</groupId> <artifactId>spotbugs</artifactId> - <version>${spotbugs.version}</version> + <version>${spotbugs.version}</version> </dependency> </dependencies> <configuration>