This is an automated email from the ASF dual-hosted git repository.
fchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 6b0addb93 [CELEBORN-989] Add support for making distribution package
via SBT
6b0addb93 is described below
commit 6b0addb93484b5c78b47d194251838748e4dd67f
Author: Fu Chen <[email protected]>
AuthorDate: Wed Sep 20 10:03:01 2023 +0800
[CELEBORN-989] Add support for making distribution package via SBT
### What changes were proposed in this pull request?
As title
### Why are the changes needed?
As title
### Does this PR introduce _any_ user-facing change?
Users have the capability to generate the binary distribution package using
SBT by executing the following command:
```shell
./build/make-distribution.sh --sbt-enabled
```
### How was this patch tested?
Pass GA && locally tested.
Closes #1921 from cfmcgrady/sbt-make-dist-3.
Authored-by: Fu Chen <[email protected]>
Signed-off-by: Fu Chen <[email protected]>
---
build/make-distribution.sh | 166 ++++++++++++++++++++++++++++++++++----------
project/BuildTools.scala | 82 ++++++++++++++++++++++
project/CelebornBuild.scala | 9 ++-
3 files changed, 220 insertions(+), 37 deletions(-)
diff --git a/build/make-distribution.sh b/build/make-distribution.sh
index 42a2d20ba..269808344 100755
--- a/build/make-distribution.sh
+++ b/build/make-distribution.sh
@@ -25,12 +25,14 @@ DIST_DIR="$PROJECT_DIR/dist"
NAME="bin"
RELEASE="false"
MVN="$PROJECT_DIR/build/mvn"
+SBT="$PROJECT_DIR/build/sbt"
+SBT_ENABLED="false"
function exit_with_usage {
echo "make-distribution.sh - tool for making binary distributions of
Celeborn"
echo ""
echo "usage:"
- cl_options="[--name <custom_name>] [--release] [--mvn <mvn-command>]"
+ cl_options="[--name <custom_name>] [--release] [--sbt-enabled] [--mvn
<mvn-command>]"
echo "make-distribution.sh $cl_options <maven build options>"
echo ""
exit 1
@@ -50,6 +52,9 @@ while (( "$#" )); do
--release)
RELEASE="true"
;;
+ --sbt-enabled)
+ SBT_ENABLED="true"
+ ;;
--help)
exit_with_usage
;;
@@ -236,42 +241,133 @@ function build_mr_client {
cp
"$PROJECT_DIR"/client-mr/mr-shaded/target/celeborn-client-mr-shaded_${SCALA_VERSION}-$VERSION.jar
"$DIST_DIR/mr/"
}
-if [ "$RELEASE" == "true" ]; then
- build_service
- build_spark_client -Pspark-2.4
- build_spark_client -Pspark-3.4
- build_flink_client -Pflink-1.14
- build_flink_client -Pflink-1.15
- build_flink_client -Pflink-1.17
- build_mr_client mr
+
+#########################
+# sbt functions #
+#########################
+
+function sbt_build_service {
+ VERSION=$("$SBT" "Show / version" | awk '/\[info\]/{ver=$2} END{print ver}')
+
+ SCALA_VERSION=$("$SBT" "Show / scalaBinaryVersion" | awk '/\[info\]/{ver=$2}
END{print ver}')
+
+ echo "Celeborn version is $VERSION"
+ echo "Making apache-celeborn-$VERSION-$NAME.tgz"
+
+ echo "Celeborn $VERSION$GITREVSTRING" > "$DIST_DIR/RELEASE"
+ echo "Build flags: $@" >> "$DIST_DIR/RELEASE"
+
+ BUILD_COMMAND=("$SBT" clean package $@)
+
+ # Actually build the jar
+ echo -e "\nBuilding with..."
+ echo -e "\$ ${BUILD_COMMAND[@]}\n"
+
+ "${BUILD_COMMAND[@]}"
+
+ $SBT "celeborn-master/copyJars;celeborn-worker/copyJars"
+
+ mkdir -p "$DIST_DIR/jars"
+ mkdir -p "$DIST_DIR/master-jars"
+ mkdir -p "$DIST_DIR/worker-jars"
+
+ ## Copy master jars
+ cp
"$PROJECT_DIR"/master/target/scala-$SCALA_VERSION/celeborn-master_$SCALA_VERSION-$VERSION.jar
"$DIST_DIR/master-jars/"
+ cp "$PROJECT_DIR"/master/target/scala-$SCALA_VERSION/jars/*.jar
"$DIST_DIR/jars/"
+ for jar in $(ls "$PROJECT_DIR/master/target/scala-$SCALA_VERSION/jars"); do
+ (cd $DIST_DIR/master-jars; ln -snf "../jars/$jar" .)
+ done
+ ## Copy worker jars
+ cp
"$PROJECT_DIR"/worker/target/scala-$SCALA_VERSION/celeborn-worker_$SCALA_VERSION-$VERSION.jar
"$DIST_DIR/worker-jars/"
+ cp "$PROJECT_DIR"/worker/target/scala-$SCALA_VERSION/jars/*.jar
"$DIST_DIR/jars/"
+ for jar in $(ls "$PROJECT_DIR/worker/target/scala-$SCALA_VERSION/jars"); do
+ (cd $DIST_DIR/worker-jars; ln -snf "../jars/$jar" .)
+ done
+}
+
+function sbt_build_client {
+ PROFILE="$1"
+ # get the client shaded project
+ CLIENT_PROJECT=$("$SBT" "$PROFILE" projects | grep shaded | awk '{print$2}')
+ # get the shaded jar file path
+ ASSEMBLY_OUTPUT_PATH=$("$SBT" "$PROFILE" "show
$CLIENT_PROJECT/assembly/assemblyOutputPath" | awk '/\[info\]/{ver=$2}
END{print ver}')
+ # build the shaded jar
+ $SBT $PROFILE "clean;$CLIENT_PROJECT/assembly"
+
+ PROFILE_PREFIX=$(echo "$PROFILE" | cut -d'-' -f2)
+ CLIENT_SUB_DIR="${PROFILE_PREFIX:1}"
+ mkdir -p "$DIST_DIR/$CLIENT_SUB_DIR"
+ cp "$ASSEMBLY_OUTPUT_PATH" "$DIST_DIR/$CLIENT_SUB_DIR/"
+}
+
+
+if [ "$SBT_ENABLED" == "true" ]; then
+ sbt_build_service "$@"
+ if [ "$RELEASE" == "true" ]; then
+ sbt_build_client -Pspark-2.4
+ sbt_build_client -Pspark-3.4
+ sbt_build_client -Pflink-1.14
+ sbt_build_client -Pflink-1.15
+ sbt_build_client -Pflink-1.17
+ else
+ echo "build client with $@"
+ ENGINE_COUNT=0
+ ENGINES=("spark" "flink" "mr")
+ for single_engine in ${ENGINES[@]}
+ do
+ echo $single_engine
+ if [[ $@ == *"${single_engine}"* ]];then
+ ENGINE_COUNT=`expr ${ENGINE_COUNT} + 1`
+ fi
+ done
+ if [[ ${ENGINE_COUNT} -eq 0 ]]; then
+ echo "Skip building client."
+ elif [[ ${ENGINE_COUNT} -ge 2 ]]; then
+ echo "Error: unsupported build options: $@"
+ echo " currently we do not support compiling different engine
clients at the same time."
+ exit -1
+ else
+ sbt_build_client $@
+ fi
+ fi
else
- ## build release package on demand
- build_service $@
- echo "build client with $@"
- ENGINE_COUNT=0
- ENGINES=("spark" "flink" "mr")
- for single_engine in ${ENGINES[@]}
- do
- echo $single_engine
- if [[ $@ == *"${single_engine}"* ]];then
- ENGINE_COUNT=`expr ${ENGINE_COUNT} + 1`
+ if [ "$RELEASE" == "true" ]; then
+ build_service
+ build_spark_client -Pspark-2.4
+ build_spark_client -Pspark-3.4
+ build_flink_client -Pflink-1.14
+ build_flink_client -Pflink-1.15
+ build_flink_client -Pflink-1.17
+ build_mr_client mr
+ else
+ ## build release package on demand
+ build_service $@
+ echo "build client with $@"
+ ENGINE_COUNT=0
+ ENGINES=("spark" "flink" "mr")
+ for single_engine in ${ENGINES[@]}
+ do
+ echo $single_engine
+ if [[ $@ == *"${single_engine}"* ]];then
+ ENGINE_COUNT=`expr ${ENGINE_COUNT} + 1`
+ fi
+ done
+ if [[ ${ENGINE_COUNT} -eq 0 ]]; then
+ echo "Skip building client."
+ elif [[ ${ENGINE_COUNT} -ge 2 ]]; then
+ echo "Error: unsupported build options: $@"
+ echo " currently we do not support compiling different engine
clients at the same time."
+ exit -1
+ elif [[ $@ == *"spark"* ]]; then
+ echo "build spark clients"
+ build_spark_client $@
+ elif [[ $@ == *"flink"* ]]; then
+ echo "build flink clients"
+ build_flink_client $@
+ elif [[ $@ == *"mr"* ]]; then
+ echo "build mr clients"
+ build_mr_client $@
fi
- done
- if [[ ${ENGINE_COUNT} -eq 0 ]]; then
- echo "Skip building client."
- elif [[ ${ENGINE_COUNT} -ge 2 ]]; then
- echo "Error: unsupported build options: $@"
- echo " currently we do not support compiling different engine
clients at the same time."
- exit -1
- elif [[ $@ == *"spark"* ]]; then
- echo "build spark clients"
- build_spark_client $@
- elif [[ $@ == *"flink"* ]]; then
- echo "build flink clients"
- build_flink_client $@
- elif [[ $@ == *"mr"* ]]; then
- echo "build mr clients"
- build_mr_client $@
fi
fi
diff --git a/project/BuildTools.scala b/project/BuildTools.scala
new file mode 100644
index 000000000..7ed25596a
--- /dev/null
+++ b/project/BuildTools.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.build
+
+import java.nio.file.Files
+
+import sbt._
+import Keys._
+
+/**
+ * This is a SBT AutoPlugin for managing distribution jar files.
+ *
+ * Usage:
+ * 1. into sub project.
+ * {{
+ * project celeborn-worker
+ * }}
+ * 2. Configure the `distributionOutputPath` setting to specify the target
directory for distribution jars.
+ * {{
+ * set distributionOutputPath := Option(new File("/path/to/jars"))
+ * }}
+ * 3. Use the `copyJars` task to copy jar files to the distribution directory.
+ * {{
+ * copyJars
+ * }}
+ */
+object DistributionToolsPlugin extends AutoPlugin {
+ override def trigger = allRequirements
+
+ object autoImport {
+ val distributionOutputPath = taskKey[Option[File]]("Path for outputting
distribution jars.")
+ val copyJars = taskKey[Unit]("Copy jars to the specified path.")
+ }
+
+ import autoImport._
+
+ override lazy val globalSettings: Seq[Setting[_]] = Seq(
+ distributionOutputPath := None
+ )
+
+ override lazy val projectSettings: Seq[Setting[_]] = Seq(
+ copyJars := {
+ val log = streams.value.log
+ val outputPath = distributionOutputPath.value.getOrElse {
+ crossTarget.value / "jars"
+ }
+
+ log.info(s"Copying jars to target directory: $outputPath")
+
+ Files.createDirectories(outputPath.toPath)
+
+ // Copy internal dependency jars
+ // Utilize the `Compile` scope to exclude the dependency of the project
itself
+ (Compile / internalDependencyAsJars).value.files.foreach { jarFile =>
+ log.info(s"Copying internal dependency jar: ${jarFile.getName}")
+ IO.copyFile(jarFile, outputPath / jarFile.getName)
+ }
+
+ // Copy managed classpath jars
+ (Runtime / managedClasspath).value.files.foreach { jarFile =>
+ log.info(s"Copying dependency jar: ${jarFile.getName}")
+ IO.copyFile(jarFile, outputPath / jarFile.getName)
+ }
+ }
+ )
+}
+
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 9188e42c3..31d3c836e 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -394,10 +394,15 @@ object CelebornWorker {
lazy val worker = Project("celeborn-worker", file("worker"))
.dependsOn(CelebornService.service)
.dependsOn(CelebornCommon.common % "test->test;compile->compile")
- .dependsOn(CelebornClient.client % "test->test;compile->compile")
- .dependsOn(CelebornMaster.master % "test->test;compile->compile")
+ .dependsOn(CelebornClient.client % "test->compile")
+ .dependsOn(CelebornMaster.master % "test->compile")
.settings (
commonSettings,
+ excludeDependencies ++= Seq(
+ // ratis-common/ratis-client are the transitive dependencies from
celeborn-common
+ ExclusionRule("org.apache.ratis", "ratis-common"),
+ ExclusionRule("org.apache.ratis", "ratis-client")
+ ),
libraryDependencies ++= Seq(
Dependencies.guava,
Dependencies.commonsIo,
