Repository: spark Updated Branches: refs/heads/master ba838fee0 -> 1c0423b28
[SPARK-25445][BUILD] the release script should be able to publish a scala-2.12 build ## What changes were proposed in this pull request? update the package and publish steps, to support scala 2.12 ## How was this patch tested? manual test Closes #22441 from cloud-fan/scala. Authored-by: Wenchen Fan <[email protected]> Signed-off-by: Wenchen Fan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1c0423b2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1c0423b2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1c0423b2 Branch: refs/heads/master Commit: 1c0423b28705eb96237c0cb4e90f49305c64a997 Parents: ba838fe Author: Wenchen Fan <[email protected]> Authored: Tue Sep 18 22:29:00 2018 +0800 Committer: Wenchen Fan <[email protected]> Committed: Tue Sep 18 22:29:00 2018 +0800 ---------------------------------------------------------------------- dev/create-release/release-build.sh | 49 +++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/1c0423b2/dev/create-release/release-build.sh ---------------------------------------------------------------------- diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 4753c29..4c90a77 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -111,13 +111,17 @@ fi # different versions of Scala are supported. BASE_PROFILES="-Pmesos -Pyarn" PUBLISH_SCALA_2_10=0 +PUBLISH_SCALA_2_12=0 SCALA_2_10_PROFILES="-Pscala-2.10" SCALA_2_11_PROFILES= -SCALA_2_12_PROFILES="-Pscala-2.12" +SCALA_2_12_PROFILES="-Pscala-2.12 -Pkafka-0-8" if [[ $SPARK_VERSION > "2.3" ]]; then BASE_PROFILES="$BASE_PROFILES -Pkubernetes -Pflume" SCALA_2_11_PROFILES="-Pkafka-0-8" + if [[ $SPARK_VERSION > "2.4" ]]; then + PUBLISH_SCALA_2_12=1 + fi else PUBLISH_SCALA_2_10=1 fi @@ -186,8 +190,17 @@ if [[ "$1" == "package" ]]; then # Updated for each binary build make_binary_release() { NAME=$1 - FLAGS="$MVN_EXTRA_OPTS -B $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES $2" - BUILD_PACKAGE=$3 + SCALA_VERSION=$2 + SCALA_PROFILES= + if [[ SCALA_VERSION == "2.10" ]]; then + SCALA_PROFILES="$SCALA_2_10_PROFILES" + elif [[ SCALA_VERSION == "2.12" ]]; then + SCALA_PROFILES="$SCALA_2_12_PROFILES" + else + SCALA_PROFILES="$SCALA_2_11_PROFILES" + fi + FLAGS="$MVN_EXTRA_OPTS -B $SCALA_PROFILES $BASE_RELEASE_PROFILES $3" + BUILD_PACKAGE=$4 # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds # share the same Zinc server. @@ -197,10 +210,11 @@ if [[ "$1" == "package" ]]; then cp -r spark spark-$SPARK_VERSION-bin-$NAME cd spark-$SPARK_VERSION-bin-$NAME - # TODO There should probably be a flag to make-distribution to allow 2.12 support - #if [[ $FLAGS == *scala-2.12* ]]; then - # ./dev/change-scala-version.sh 2.12 - #fi + if [[ SCALA_VERSION == "2.10" ]]; then + ./dev/change-scala-version.sh 2.10 + elif [[ SCALA_VERSION == "2.12" ]]; then + ./dev/change-scala-version.sh 2.12 + fi export ZINC_PORT=$ZINC_PORT echo "Creating distribution: $NAME ($FLAGS)" @@ -291,11 +305,20 @@ if [[ "$1" == "package" ]]; then for key in ${!BINARY_PKGS_ARGS[@]}; do args=${BINARY_PKGS_ARGS[$key]} extra=${BINARY_PKGS_EXTRA[$key]} - if ! make_binary_release "$key" "$args" "$extra"; then + if ! make_binary_release "$key" "2.11" "$args" "$extra"; then error "Failed to build $key package. Check logs for details." fi done + if [[ $PUBLISH_SCALA_2_12 = 1 ]]; then + key="without-hadoop-scala-2.12" + args="-Phadoop-provided" + extra="" + if ! make_binary_release "$key" "2.12" "$args" "$extra"; then + error "Failed to build $key package. Check logs for details." + fi + fi + rm -rf spark-$SPARK_VERSION-bin-*/ if ! is_dry_run; then @@ -414,15 +437,15 @@ if [[ "$1" == "publish-release" ]]; then -DskipTests $PUBLISH_PROFILES $SCALA_2_10_PROFILES clean install fi - #./dev/change-scala-version.sh 2.12 - #$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo \ - # -DskipTests $SCALA_2_12_PROFILES §$PUBLISH_PROFILES clean install + if ! is_dry_run && [[ $PUBLISH_SCALA_2_12 = 1 ]]; then + ./dev/change-scala-version.sh 2.12 + $MVN -DzincPort=$((ZINC_PORT + 2)) -Dmaven.repo.local=$tmp_repo -Dscala-2.12 \ + -DskipTests $PUBLISH_PROFILES $SCALA_2_12_PROFILES clean install + fi # Clean-up Zinc nailgun process $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill - #./dev/change-scala-version.sh 2.11 - pushd $tmp_repo/org/apache/spark # Remove any extra files generated during install --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
