This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2087b5729 [GLUTEN-5777][VL] Supporting specify spark version when
build (#5778)
2087b5729 is described below
commit 2087b57293d455c50c3b067108117bf5437573a4
Author: James Xu <[email protected]>
AuthorDate: Fri May 17 13:31:10 2024 +0800
[GLUTEN-5777][VL] Supporting specify spark version when build (#5778)
Support a new option: --spark_version for buildbundle-veloxbe.sh and
builddeps-veloxbe.sh. By default the version is ALL which keeps the same
behaviour as now.
---
dev/buildbundle-veloxbe.sh | 19 ++++++++++++----
dev/builddeps-veloxbe.sh | 14 ++++++++++++
docs/get-started/build-guide.md | 49 +++++++++++++++++++++--------------------
3 files changed, 54 insertions(+), 28 deletions(-)
diff --git a/dev/buildbundle-veloxbe.sh b/dev/buildbundle-veloxbe.sh
index 10c8e61e2..eaa82730b 100755
--- a/dev/buildbundle-veloxbe.sh
+++ b/dev/buildbundle-veloxbe.sh
@@ -3,8 +3,19 @@
BASEDIR=$(dirname $0)
source "$BASEDIR/builddeps-veloxbe.sh"
+function build_for_spark {
+ spark_version=$1
+ mvn clean package -Pbackends-velox -Pceleborn -Puniffle
-Pspark-$spark_version -DskipTests
+}
+
cd $GLUTEN_DIR
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.4 -DskipTests
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests
+
+# SPARK_VERSION is defined in builddeps-veloxbe.sh
+if [ "$SPARK_VERSION" = "ALL" ]; then
+ for spark_version in 3.2 3.3 3.4 3.5
+ do
+ build_for_spark $spark_version
+ done
+else
+ build_for_spark $SPARK_VERSION
+fi
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 232c36d40..35976d37a 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -32,6 +32,7 @@ VELOX_BRANCH=""
VELOX_HOME=""
VELOX_PARAMETER=""
COMPILE_ARROW_JAVA=ON
+SPARK_VERSION=ALL
# set default number of threads as cpu cores minus 2
if [[ "$(uname)" == "Darwin" ]]; then
@@ -142,6 +143,10 @@ do
--num_threads=*)
NUM_THREADS=("${arg#*=}")
shift # Remove argument name from processing
+ ;;
+ --spark_version=*)
+ SPARK_VERSION=("${arg#*=}")
+ shift # Remove argument name from processing
;;
*)
OTHER_ARGUMENTS+=("$1")
@@ -173,6 +178,15 @@ if [ "$ENABLE_VCPKG" = "ON" ]; then
eval "$envs"
fi
+if [ "$SPARK_VERSION" = "3.2" ] || [ "$SPARK_VERSION" = "3.3" ] \
+ || [ "$SPARK_VERSION" = "3.4" ] || [ "$SPARK_VERSION" = "3.5" ] \
+ || [ "$SPARK_VERSION" = "ALL" ]; then
+ echo "Building for Spark $SPARK_VERSION"
+else
+ echo "Invalid Spark version: $SPARK_VERSION"
+ exit 1
+fi
+
concat_velox_param
function build_velox {
diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md
index 90c2ed119..3db2244ba 100644
--- a/docs/get-started/build-guide.md
+++ b/docs/get-started/build-guide.md
@@ -8,30 +8,31 @@ parent: Getting-Started
### Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh
Please set them via `--`, e.g. `--build_type=Release`.
-| Parameters | Description
| Default |
-|------------------------|----------------------------------------------------------------------------|---------|
-| build_type | Build type for Velox & gluten cpp,
CMAKE_BUILD_TYPE. | Release |
-| build_tests | Build gluten cpp tests.
| OFF |
-| build_examples | Build udf example.
| OFF |
-| build_benchmarks | Build gluten cpp benchmarks.
| OFF |
-| build_jemalloc | Build with jemalloc.
| ON |
-| build_protobuf | Build protobuf lib.
| ON |
-| enable_qat | Enable QAT for shuffle data de/compression.
| OFF |
-| enable_iaa | Enable IAA for shuffle data de/compression.
| OFF |
-| enable_hbm | Enable HBM allocator.
| OFF |
-| enable_s3 | Build with S3 support.
| OFF |
-| enable_gcs | Build with GCs support.
| OFF |
-| enable_hdfs | Build with HDFS support.
| OFF |
-| enable_abfs | Build with ABFS support.
| OFF |
-| enable_ep_cache | Enable caching for external project build (Velox).
| OFF |
-| enable_vcpkg | Enable vcpkg for static build.
| OFF |
-| run_setup_script | Run setup script to install Velox dependencies.
| ON |
-| velox_repo | Specify your own Velox repo to build.
| "" |
-| velox_branch | Specify your own Velox branch to build.
| "" |
-| velox_home | Specify your own Velox source path to build.
| "" |
-| build_velox_tests | Build Velox tests.
| OFF |
-| build_velox_benchmarks | Build Velox benchmarks (velox_tests and connectors
will be disabled if ON) | OFF |
-| compile_arrow_java | Compile arrow java for gluten build to use to fix
invalid pointer issues. | ON |
+| Parameters | Description
| Default |
+|------------------------|----------------------------------------------------------------------------------------------------|---------|
+| build_type | Build type for Velox & gluten cpp,
CMAKE_BUILD_TYPE. | Release |
+| build_tests | Build gluten cpp tests.
| OFF |
+| build_examples | Build udf example.
| OFF |
+| build_benchmarks | Build gluten cpp benchmarks.
| OFF |
+| build_jemalloc | Build with jemalloc.
| ON |
+| build_protobuf | Build protobuf lib.
| ON |
+| enable_qat | Enable QAT for shuffle data de/compression.
| OFF |
+| enable_iaa | Enable IAA for shuffle data de/compression.
| OFF |
+| enable_hbm | Enable HBM allocator.
| OFF |
+| enable_s3 | Build with S3 support.
| OFF |
+| enable_gcs | Build with GCs support.
| OFF |
+| enable_hdfs | Build with HDFS support.
| OFF |
+| enable_abfs | Build with ABFS support.
| OFF |
+| enable_ep_cache | Enable caching for external project build (Velox).
| OFF |
+| enable_vcpkg | Enable vcpkg for static build.
| OFF |
+| run_setup_script | Run setup script to install Velox dependencies.
| ON |
+| velox_repo | Specify your own Velox repo to build.
| "" |
+| velox_branch | Specify your own Velox branch to build.
| "" |
+| velox_home | Specify your own Velox source path to build.
| "" |
+| build_velox_tests | Build Velox tests.
| OFF |
+| build_velox_benchmarks | Build Velox benchmarks (velox_tests and connectors
will be disabled if ON) | OFF |
+| compile_arrow_java | Compile arrow java for gluten build to use to fix
invalid pointer issues. | ON |
+| spark_version | Build for specified version of Spark(3.2, 3.3, 3.4,
3.5, ALL). `ALL` means build for all versions. | ALL |
### Velox build parameters for build_velox.sh
Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
