This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2b99fff532 [VL] Remove compile option ENABLE_EP_CACHE (#8350)
2b99fff532 is described below
commit 2b99fff532343e91d6e96008d20b013b8ee78156
Author: Hongze Zhang <[email protected]>
AuthorDate: Thu Dec 26 16:13:43 2024 +0800
[VL] Remove compile option ENABLE_EP_CACHE (#8350)
---
dev/builddeps-veloxbe.sh | 7 +--
dev/ci-velox-buildshared-centos-8.sh | 2 +-
docs/get-started/Velox.md | 4 +-
docs/get-started/build-guide.md | 2 -
ep/build-velox/src/build_velox.sh | 50 +---------------------
tools/gluten-te/centos/dockerfile-build | 2 +-
tools/gluten-te/ubuntu/dockerfile-build | 2 +-
.../examples/buildhere-veloxbe-dev/scripts/all.sh | 2 +-
8 files changed, 9 insertions(+), 62 deletions(-)
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 1ed6e62e29..8eb4cf4edf 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -22,7 +22,6 @@ ENABLE_GCS=OFF
ENABLE_S3=OFF
ENABLE_HDFS=OFF
ENABLE_ABFS=OFF
-ENABLE_EP_CACHE=OFF
ENABLE_VCPKG=OFF
RUN_SETUP_SCRIPT=ON
VELOX_REPO=""
@@ -96,10 +95,6 @@ do
ENABLE_ABFS=("${arg#*=}")
shift # Remove argument name from processing
;;
- --enable_ep_cache=*)
- ENABLE_EP_CACHE=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
--enable_vcpkg=*)
ENABLE_VCPKG=("${arg#*=}")
shift # Remove argument name from processing
@@ -192,7 +187,7 @@ function build_velox {
cd $GLUTEN_DIR/ep/build-velox/src
# When BUILD_TESTS is on for gluten cpp, we need turn on
VELOX_BUILD_TEST_UTILS via build_test_utils.
./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS
--build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \
- --enable_abfs=$ENABLE_ABFS
--enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS \
+ --enable_abfs=$ENABLE_ABFS --build_test_utils=$BUILD_TESTS \
--build_tests=$BUILD_VELOX_TESTS
--build_benchmarks=$BUILD_VELOX_BENCHMARKS --num_threads=$NUM_THREADS \
--velox_home=$VELOX_HOME
}
diff --git a/dev/ci-velox-buildshared-centos-8.sh
b/dev/ci-velox-buildshared-centos-8.sh
index ff9a62b798..3f7197912a 100755
--- a/dev/ci-velox-buildshared-centos-8.sh
+++ b/dev/ci-velox-buildshared-centos-8.sh
@@ -3,5 +3,5 @@
set -e
source /opt/rh/gcc-toolset-11/enable
-./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF
--enable_ep_cache=OFF --build_tests=ON \
+./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF
--build_tests=ON \
--build_examples=ON --build_benchmarks=ON
diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
index a0af0cc75d..48bca9a6d3 100644
--- a/docs/get-started/Velox.md
+++ b/docs/get-started/Velox.md
@@ -61,11 +61,11 @@ First time build for all supported spark versions.
./dev/buildbundle-veloxbe.sh
```
-After a complete build, if only some gluten code is changed, you can use the
following command to skip building velox/arrow and
+After a complete build, if only some gluten code is changed, you can use the
following command to skip building arrow and
setting up build dependencies.
```bash
-./dev/buildbundle-veloxbe.sh --enable_ep_cache=ON --build_arrow=OFF
--run_setup_script=OFF
+./dev/buildbundle-veloxbe.sh --build_arrow=OFF --run_setup_script=OFF
```
**For aarch64 build**
diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md
index c68a5f4633..d9c3beaab9 100644
--- a/docs/get-started/build-guide.md
+++ b/docs/get-started/build-guide.md
@@ -22,7 +22,6 @@ Please set them via `--`, e.g. `--build_type=Release`.
| enable_gcs | Build with GCS support.
| OFF |
| enable_hdfs | Build with HDFS support.
| OFF |
| enable_abfs | Build with ABFS support.
| OFF |
-| enable_ep_cache | Enable caching for external project build (Velox).
| OFF |
| enable_vcpkg | Enable vcpkg for static build.
| OFF |
| run_setup_script | Run setup script to install Velox dependencies.
| ON |
| velox_repo | Specify your own Velox repo to build.
| "" |
@@ -45,7 +44,6 @@ Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`.
| enable_hdfs | Build Velox with HDFS support.
| OFF |
| enable_abfs | Build Velox with ABFS support.
| OFF |
| run_setup_script | Run setup script to install Velox dependencies before
build. | ON |
-| enable_ep_cache | Enable and reuse cache of Velox build.
| OFF |
| build_test_utils | Build Velox with cmake arg -DVELOX_BUILD_TEST_UTILS=ON if
ON. | OFF |
| build_tests | Build Velox test.
| OFF |
| build_benchmarks | Build Velox benchmarks.
| OFF |
diff --git a/ep/build-velox/src/build_velox.sh
b/ep/build-velox/src/build_velox.sh
index a5f009833a..67a540e76f 100755
--- a/ep/build-velox/src/build_velox.sh
+++ b/ep/build-velox/src/build_velox.sh
@@ -26,7 +26,6 @@ ENABLE_HDFS=OFF
ENABLE_ABFS=OFF
BUILD_TYPE=release
VELOX_HOME=""
-ENABLE_EP_CACHE=OFF
# May be deprecated in Gluten build.
ENABLE_BENCHMARK=OFF
# May be deprecated in Gluten build.
@@ -65,10 +64,6 @@ for arg in "$@"; do
BUILD_TYPE=("${arg#*=}")
shift # Remove argument name from processing
;;
- --enable_ep_cache=*)
- ENABLE_EP_CACHE=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
--build_test_utils=*)
BUILD_TEST_UTILS=("${arg#*=}")
shift # Remove argument name from processing
@@ -167,40 +162,6 @@ function compile {
fi
}
-function get_build_summary {
- COMMIT_HASH=$1
- # Ideally all script arguments should be put into build summary.
- # ENABLE_EP_CACHE is excluded. Thus, in current build with
ENABLE_EP_CACHE=ON, we can use EP cache
- # from last build with ENABLE_EP_CACHE=OFF,
- echo
"ENABLE_S3=$ENABLE_S3,ENABLE_GCS=$ENABLE_GCS,ENABLE_HDFS=$ENABLE_HDFS,ENABLE_ABFS=$ENABLE_ABFS,\
-BUILD_TYPE=$BUILD_TYPE,VELOX_HOME=$VELOX_HOME,ENABLE_BENCHMARK=$ENABLE_BENCHMARK,\
-ENABLE_TESTS=$ENABLE_TESTS,BUILD_TEST_UTILS=$BUILD_TEST_UTILS,\
-OTHER_ARGUMENTS=$OTHER_ARGUMENTS,COMMIT_HASH=$COMMIT_HASH"
-}
-
-function check_commit {
- if [ $ENABLE_EP_CACHE == "ON" ]; then
- if [ -f ${VELOX_HOME}/velox-build.cache ]; then
- CACHED_BUILD_SUMMARY="$(cat ${VELOX_HOME}/velox-build.cache)"
- if [ -n "$CACHED_BUILD_SUMMARY" ]; then
- if [ "$TARGET_BUILD_SUMMARY" = "$CACHED_BUILD_SUMMARY" ]; then
- echo "Velox build $TARGET_BUILD_SUMMARY was cached."
- exit 0
- else
- echo "Found cached build $CACHED_BUILD_SUMMARY for Velox which is
different with target build $TARGET_BUILD_SUMMARY."
- fi
- fi
- fi
- else
- # Branch-new build requires all untracked files to be deleted. We only
need the source code.
- sudo git clean -dffx :/
- fi
-
- if [ -f ${VELOX_HOME}/velox-build.cache ]; then
- rm -f ${VELOX_HOME}/velox-build.cache
- fi
-}
-
CURRENT_DIR=$(
cd "$(dirname "$BASH_SOURCE")"
pwd
@@ -220,15 +181,8 @@ echo "ENABLE_ABFS=${ENABLE_ABFS}"
echo "BUILD_TYPE=${BUILD_TYPE}"
cd ${VELOX_HOME}
-TARGET_BUILD_SUMMARY=$(get_build_summary "$(git rev-parse --verify HEAD)")
-if [ -z "$TARGET_BUILD_SUMMARY" ]; then
- echo "Unable to parse Velox build: $TARGET_BUILD_SUMMARY."
- exit 1
-fi
-echo "Target Velox build: $TARGET_BUILD_SUMMARY"
-
-check_commit
+# Branch-new build requires all untracked files to be deleted. We only need
the source code.
+sudo git clean -dffx :/
compile
echo "Successfully built Velox from Source."
-echo $TARGET_BUILD_SUMMARY > "${VELOX_HOME}/velox-build.cache"
diff --git a/tools/gluten-te/centos/dockerfile-build
b/tools/gluten-te/centos/dockerfile-build
index fa496613b1..d8a16d475d 100644
--- a/tools/gluten-te/centos/dockerfile-build
+++ b/tools/gluten-te/centos/dockerfile-build
@@ -62,7 +62,7 @@ RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \
if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug";
else GLUTEN_BUILD_TYPE="Release"; fi; \
DEPS_INSTALL_SCRIPT="source /env.sh && bash
/opt/gluten/dev/builddeps-veloxbe.sh \
--enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON \
- --build_type=$GLUTEN_BUILD_TYPE
--enable_ep_cache=ON"; \
+ --build_type=$GLUTEN_BUILD_TYPE"; \
EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
-Pbackends-velox \
-Pceleborn \
diff --git a/tools/gluten-te/ubuntu/dockerfile-build
b/tools/gluten-te/ubuntu/dockerfile-build
index 59736c9869..073db0ba18 100644
--- a/tools/gluten-te/ubuntu/dockerfile-build
+++ b/tools/gluten-te/ubuntu/dockerfile-build
@@ -62,7 +62,7 @@ RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \
if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug";
else GLUTEN_BUILD_TYPE="Release"; fi; \
DEPS_INSTALL_SCRIPT="bash /opt/gluten/dev/builddeps-veloxbe.sh \
--enable_hdfs=ON --enable_s3=ON --enable_gcs=ON
--enable_abfs=ON \
- --build_type=$GLUTEN_BUILD_TYPE
--enable_ep_cache=ON"; \
+ --build_type=$GLUTEN_BUILD_TYPE"; \
EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
-Pbackends-velox \
-Pceleborn \
diff --git
a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
index f795b5ba9b..562828e5a3 100755
--- a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
+++ b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
@@ -26,7 +26,7 @@ EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
-Dcheckstyle.skip=true"
cd /opt/gluten
-bash dev/builddeps-veloxbe.sh --build_type=Debug --enable_ep_cache=ON
+bash dev/builddeps-veloxbe.sh --build_type=Debug
mvn clean install $EXTRA_MAVEN_OPTIONS
apt-get -y -q --no-install-recommends install firefox tmux openjdk-8-source
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
