This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/branch-0.3 by this push:
new 6fc4aa556 [CELEBORN-649][BUILD] Speed up make-distribution.sh
6fc4aa556 is described below
commit 6fc4aa5564d93c0ef2f4b0affee25ea7856f1687
Author: Cheng Pan <[email protected]>
AuthorDate: Thu Jun 8 15:09:56 2023 +0800
[CELEBORN-649][BUILD] Speed up make-distribution.sh
### What changes were proposed in this pull request?
This PR aims to improve `build/make-distribution.sh` by
- skip building javadoc and source artifacts
- skip building unnecessary modules
- allow skipping client modules
### Why are the changes needed?
Speed up the packaging process.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Tested with
```
build/make-distribution.sh
```
```
build/make-distribution.sh -Pspark-3.3
```
```
build/make-distribution.sh -Pflink-1.17
```
Closes #1561 from pan3793/CELEBORN-649.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit db66670253be41302d0a961660607d9318d49444)
Signed-off-by: Cheng Pan <[email protected]>
---
build/make-distribution.sh | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/build/make-distribution.sh b/build/make-distribution.sh
index 7aa330e08..5aef54537 100755
--- a/build/make-distribution.sh
+++ b/build/make-distribution.sh
@@ -108,6 +108,8 @@ cd "$PROJECT_DIR"
rm -rf "$DIST_DIR"
mkdir -p "$DIST_DIR"
+MVN_DIST_OPT="-DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip"
+
function build_service {
VERSION=$("$MVN" help:evaluate -Dexpression=project.version $@ 2>/dev/null \
| grep -v "INFO" \
@@ -127,7 +129,7 @@ function build_service {
# Store the command as an array because $MVN variable might have spaces in
it.
# Normal quoting tricks don't work.
# See: http://mywiki.wooledge.org/BashFAQ/050
- BUILD_COMMAND=("$MVN" clean package -DskipTests $@)
+ BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl master,worker -am $@)
# Actually build the jar
echo -e "\nBuilding with..."
@@ -171,7 +173,7 @@ function build_spark_client {
# Store the command as an array because $MVN variable might have spaces in
it.
# Normal quoting tricks don't work.
# See: http://mywiki.wooledge.org/BashFAQ/050
- BUILD_COMMAND=("$MVN" clean package -DskipTests -pl
:celeborn-client-spark-${SPARK_MAJOR_VERSION}-shaded_$SCALA_VERSION -am $@)
+ BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl
:celeborn-client-spark-${SPARK_MAJOR_VERSION}-shaded_$SCALA_VERSION -am $@)
# Actually build the jar
echo -e "\nBuilding with..."
@@ -199,7 +201,7 @@ function build_flink_client {
# Store the command as an array because $MVN variable might have spaces in
it.
# Normal quoting tricks don't work.
# See: http://mywiki.wooledge.org/BashFAQ/050
- BUILD_COMMAND=("$MVN" clean package -DskipTests -pl
:celeborn-client-flink-${FLINK_BINARY_VERSION}-shaded_$SCALA_VERSION -am $@)
+ BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl
:celeborn-client-flink-${FLINK_BINARY_VERSION}-shaded_$SCALA_VERSION -am $@)
# Actually build the jar
echo -e "\nBuilding with..."
@@ -222,12 +224,15 @@ if [ "$RELEASE" == "true" ]; then
else
## build release package on demand
build_service $@
- if [[ $@ == *"spark"* && $@ != *"flink"* ]]; then
+ if [[ $@ != *"spark"* && $@ != *"flink"* ]]; then
+ echo "Skip building client."
+ elif [[ $@ == *"spark"* && $@ != *"flink"* ]]; then
build_spark_client $@
elif [[ $@ == *"flink"* && $@ != *"spark"* ]]; then
build_flink_client $@
else
- echo "Error: unsupported command $@, currently we do not support compiling
spark and flink at the same time."
+ echo "Error: unsupported build options: $@"
+ echo " currently we do not support compiling Spark and Flink clients
at the same time."
exit -1
fi
fi
