[incubator-celeborn] branch main updated: [CELEBORN-150][BUILD] Reduce binary tarball size by sharing jars (#1095)

[chengpan]

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new dc663699 [CELEBORN-150][BUILD] Reduce binary tarball size by sharing 
jars (#1095)
dc663699 is described below

commit dc66369973c67d0c06250084aff8efb9aad96a4e
Author: Cheng Pan <cheng...@apache.org>
AuthorDate: Fri Dec 16 14:30:17 2022 +0800

    [CELEBORN-150][BUILD] Reduce binary tarball size by sharing jars (#1095)
    
    * [CELEBORN-150][BUILD] Reduce binary tarball size by sharing jars
    
    * nit
    
    * nit
    
    * docker
    
    * nit
    
    * cp -R
---
 dev/make-distribution.sh | 16 +++++++++++++---
 docker/Dockerfile        |  3 ++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
index 61a7147e..ef62d0c0 100755
--- a/dev/make-distribution.sh
+++ b/dev/make-distribution.sh
@@ -135,6 +135,7 @@ echo -e "\$ ${BUILD_COMMAND[@]}\n"
 
 # Make directories
 rm -rf "$DIST_DIR"
+mkdir -p "$DIST_DIR/jars"
 mkdir -p "$DIST_DIR/master-jars"
 mkdir -p "$DIST_DIR/worker-jars"
 mkdir -p "$DIST_DIR/spark"
@@ -143,10 +144,19 @@ echo "Celeborn $VERSION$GITREVSTRING" > 
"$DIST_DIR/RELEASE"
 echo "Build flags: $@" >> "$DIST_DIR/RELEASE"
 
 # Copy jars
+## Copy master jars
 cp "$PROJECT_DIR"/master/target/celeborn-master_$SCALA_VERSION-$VERSION.jar 
"$DIST_DIR/master-jars/"
-cp "$PROJECT_DIR"/master/target/scala-$SCALA_VERSION/jars/*.jar 
"$DIST_DIR/master-jars/"
+cp "$PROJECT_DIR"/master/target/scala-$SCALA_VERSION/jars/*.jar 
"$DIST_DIR/jars/"
+for jar in $(ls "$PROJECT_DIR/master/target/scala-$SCALA_VERSION/jars"); do
+  (cd $DIST_DIR/master-jars; ln -snf "../jars/$jar" .)
+done
+## Copy worker jars
 cp "$PROJECT_DIR"/worker/target/celeborn-worker_$SCALA_VERSION-$VERSION.jar 
"$DIST_DIR/worker-jars/"
-cp "$PROJECT_DIR"/worker/target/scala-$SCALA_VERSION/jars/*.jar 
"$DIST_DIR/worker-jars/"
+cp "$PROJECT_DIR"/worker/target/scala-$SCALA_VERSION/jars/*.jar 
"$DIST_DIR/jars/"
+for jar in $(ls "$PROJECT_DIR/worker/target/scala-$SCALA_VERSION/jars"); do
+  (cd $DIST_DIR/worker-jars; ln -snf "../jars/$jar" .)
+done
+## Copy spark client jars
 cp 
"$PROJECT_DIR"/client-spark/spark-$SPARK_MAJOR_VERSION-shaded/target/celeborn-client-spark-${SPARK_MAJOR_VERSION}-shaded_$SCALA_VERSION-$VERSION.jar
 "$DIST_DIR/spark/"
 
 # Copy other things
@@ -161,6 +171,6 @@ cp -r "$PROJECT_DIR/docker/helm" "$DIST_DIR/docker"
 TARDIR_NAME="apache-celeborn-$VERSION-$NAME"
 TARDIR="$PROJECT_DIR/$TARDIR_NAME"
 rm -rf "$TARDIR"
-cp -r "$DIST_DIR" "$TARDIR"
+cp -R "$DIST_DIR" "$TARDIR"
 tar czf "apache-celeborn-$VERSION-$NAME.tgz" -C "$PROJECT_DIR" "$TARDIR_NAME"
 rm -rf "$TARDIR"
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7ae4d234..f799b8d4 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -30,6 +30,7 @@ RUN set -ex && \
 COPY bin /opt/celeborn/bin
 COPY sbin /opt/celeborn/sbin
 COPY conf /opt/celeborn/conf
-COPY RELEASE /opt/celeborn/RELEASE
+COPY jars /opt/celeborn/jars
 COPY master-jars /opt/celeborn/master-jars
 COPY worker-jars /opt/celeborn/worker-jars
+COPY RELEASE /opt/celeborn/RELEASE