Repository: spark Updated Branches: refs/heads/master bfb74394a -> 717f58e9c
[SPARK-24685][BUILD] Restore support for building old Hadoop versions of 2.1. Update the release scripts to build binary packages for older versions of Hadoop when building Spark 2.1. Also did some minor refactoring of that part of the script so that changing these later is easier. This was used to build the missing packages from 2.1.3-rc2. Author: Marcelo Vanzin <[email protected]> Closes #21661 from vanzin/SPARK-24685. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/717f58e9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/717f58e9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/717f58e9 Branch: refs/heads/master Commit: 717f58e9ced9b43940719dfc8675216009a4c2e9 Parents: bfb7439 Author: Marcelo Vanzin <[email protected]> Authored: Wed Aug 15 14:42:48 2018 -0700 Committer: Marcelo Vanzin <[email protected]> Committed: Wed Aug 15 14:42:48 2018 -0700 ---------------------------------------------------------------------- dev/create-release/release-build.sh | 50 +++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/717f58e9/dev/create-release/release-build.sh ---------------------------------------------------------------------- diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 24a62a8..73610a3 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -178,12 +178,17 @@ if [[ "$1" == "package" ]]; then SHA512 spark-$SPARK_VERSION.tgz > spark-$SPARK_VERSION.tgz.sha512 rm -rf spark-$SPARK_VERSION + ZINC_PORT=3035 + # Updated for each binary build make_binary_release() { NAME=$1 - FLAGS=$2 - ZINC_PORT=$3 - BUILD_PACKAGE=$4 + FLAGS="$MVN_EXTRA_OPTS -B $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES $2" + BUILD_PACKAGE=$3 + + # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds + # share the same Zinc server. + ZINC_PORT=$((ZINC_PORT + 1)) echo "Building binary dist $NAME" cp -r spark spark-$SPARK_VERSION-bin-$NAME @@ -255,20 +260,39 @@ if [[ "$1" == "package" ]]; then spark-$SPARK_VERSION-bin-$NAME.tgz.sha512 } - # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds - # share the same Zinc server. - if ! make_binary_release "hadoop2.6" "$MVN_EXTRA_OPTS -B -Phadoop-2.6 $HIVE_PROFILES $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3035" "withr"; then - error "Failed to build hadoop2.6 package. Check logs for details." - fi + # List of binary packages built. Populates two associative arrays, where the key is the "name" of + # the package being built, and the values are respectively the needed maven arguments for building + # the package, and any extra package needed for that particular combination. + # + # In dry run mode, only build the first one. The keys in BINARY_PKGS_ARGS are used as the + # list of packages to be built, so it's ok for things to be missing in BINARY_PKGS_EXTRA. + + declare -A BINARY_PKGS_ARGS + BINARY_PKGS_ARGS["hadoop2.7"]="-Phadoop-2.7 $HIVE_PROFILES" if ! is_dry_run; then - if ! make_binary_release "hadoop2.7" "$MVN_EXTRA_OPTS -B -Phadoop-2.7 $HIVE_PROFILES $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3036" "withpip"; then - error "Failed to build hadoop2.7 package. Check logs for details." - fi - if ! make_binary_release "without-hadoop" "$MVN_EXTRA_OPTS -B -Phadoop-provided $SCALA_2_11_PROFILES $BASE_RELEASE_PROFILES" "3037"; then - error "Failed to build without-hadoop package. Check logs for details." + BINARY_PKGS_ARGS["hadoop2.6"]="-Phadoop-2.6 $HIVE_PROFILES" + BINARY_PKGS_ARGS["without-hadoop"]="-Pwithout-hadoop" + if [[ $SPARK_VERSION < "2.2." ]]; then + BINARY_PKGS_ARGS["hadoop2.4"]="-Phadoop-2.4 $HIVE_PROFILES" + BINARY_PKGS_ARGS["hadoop2.3"]="-Phadoop-2.3 $HIVE_PROFILES" fi fi + declare -A BINARY_PKGS_EXTRA + BINARY_PKGS_EXTRA["hadoop2.7"]="withpip" + if ! is_dry_run; then + BINARY_PKGS_EXTRA["hadoop2.6"]="withr" + fi + + echo "Packages to build: ${!BINARY_PKGS_ARGS[@]}" + for key in ${!BINARY_PKGS_ARGS[@]}; do + args=${BINARY_PKGS_ARGS[$key]} + extra=${BINARY_PKGS_EXTRA[$key]} + if ! make_binary_release "$key" "$args" "$extra"; then + error "Failed to build $key package. Check logs for details." + fi + done + rm -rf spark-$SPARK_VERSION-bin-*/ if ! is_dry_run; then --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
