This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7a3c129c5 [VL] Disable protobuf build by default (#6297)
7a3c129c5 is described below
commit 7a3c129c5eb1359dfa4d8346373ca0eca89c6db8
Author: PHILO-HE <[email protected]>
AuthorDate: Tue Jul 2 10:31:31 2024 +0800
[VL] Disable protobuf build by default (#6297)
---
dev/builddeps-veloxbe.sh | 4 ++--
docs/get-started/build-guide.md | 4 ++--
ep/build-velox/src/get_velox.sh | 23 +++++------------------
3 files changed, 9 insertions(+), 22 deletions(-)
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index d5e33e926..4e0882a83 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -13,7 +13,7 @@ BUILD_TESTS=OFF
BUILD_EXAMPLES=OFF
BUILD_BENCHMARKS=OFF
BUILD_JEMALLOC=OFF
-BUILD_PROTOBUF=ON
+BUILD_PROTOBUF=OFF
BUILD_VELOX_TESTS=OFF
BUILD_VELOX_BENCHMARKS=OFF
ENABLE_QAT=OFF
@@ -201,7 +201,7 @@ function build_arrow {
function build_velox {
echo "Start to build Velox"
cd $GLUTEN_DIR/ep/build-velox/src
- ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --build_protobuf=$BUILD_PROTOBUF
--enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS
$VELOX_PARAMETER
+ ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --enable_s3=$ENABLE_S3
--enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER
# When BUILD_TESTS is on for gluten cpp, we need turn on
VELOX_BUILD_TEST_UTILS via build_test_utils.
./build_velox.sh --run_setup_script=$RUN_SETUP_SCRIPT --enable_s3=$ENABLE_S3
--enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \
--enable_abfs=$ENABLE_ABFS
--enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS
--build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS \
diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md
index dc4989bc8..a9e9bd126 100644
--- a/docs/get-started/build-guide.md
+++ b/docs/get-started/build-guide.md
@@ -14,8 +14,8 @@ Please set them via `--`, e.g. `--build_type=Release`.
| build_tests | Build gluten cpp tests.
| OFF |
| build_examples | Build udf example.
| OFF |
| build_benchmarks | Build gluten cpp benchmarks.
| OFF |
-| build_jemalloc | Build with jemalloc.
| OFF |
-| build_protobuf | Build protobuf lib.
| ON |
+| build_jemalloc | Build with jemalloc.
| OFF |
+| build_protobuf | Build protobuf lib.
| OFF |
| enable_qat | Enable QAT for shuffle data de/compression.
| OFF |
| enable_iaa | Enable IAA for shuffle data de/compression.
| OFF |
| enable_hbm | Enable HBM allocator.
| OFF |
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 808e48881..bbc147e55 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -22,8 +22,6 @@ VELOX_HOME=""
#Set on run gluten on HDFS
ENABLE_HDFS=OFF
-#It can be set to OFF when compiling velox again
-BUILD_PROTOBUF=ON
#Set on run gluten on S3
ENABLE_S3=OFF
#Set on run gluten on GCS
@@ -47,10 +45,6 @@ for arg in "$@"; do
VELOX_HOME=("${arg#*=}")
shift # Remove argument name from processing
;;
- --build_protobuf=*)
- BUILD_PROTOBUF=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
--enable_hdfs=*)
ENABLE_HDFS=("${arg#*=}")
shift # Remove argument name from processing
@@ -95,15 +89,13 @@ function process_setup_ubuntu {
sed -i '/ccache/a\ curl \\' scripts/setup-ubuntu.sh
sed -i '/libgmock-dev/d' scripts/setup-ubuntu.sh # resolved by
ep/build-velox/build/velox_ep/CMake/resolve_dependency_modules/gtest.cmake
sed -i 's/github_checkout boostorg\/boost \"\${BOOST_VERSION}\"
--recursive/wget_and_untar
https:\/\/github.com\/boostorg\/boost\/releases\/download\/boost-1.84.0\/boost-1.84.0.tar.gz
boost \&\& cd boost/g' scripts/setup-ubuntu.sh
+ sed -i '/^function install_folly.*/i function install_protobuf {\n wget
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n
tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure
CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make
install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh
+ sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf'
scripts/setup-ubuntu.sh
if [ $ENABLE_HDFS == "ON" ]; then
sed -i '/^function install_folly.*/i function install_libhdfs3 {\n
github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n'
scripts/setup-ubuntu.sh
sed -i '/^ run_and_time install_folly/a \ \ run_and_time
install_libhdfs3' scripts/setup-ubuntu.sh
sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh
fi
- if [ $BUILD_PROTOBUF == "ON" ]; then
- sed -i '/^function install_folly.*/i function install_protobuf {\n wget
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n
tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure
CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make
install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh
- sed -i '/^ run_and_time install_folly/a \ \ run_and_time
install_protobuf' scripts/setup-ubuntu.sh
- fi
sed -i "s/apt install -y/sudo apt install -y/"
${VELOX_HOME}/scripts/setup-adapters.sh
if [ $ENABLE_S3 == "ON" ]; then
sed -i '/^ run_and_time install_folly/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh
@@ -136,15 +128,14 @@ function process_setup_centos8 {
sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel
libuuid-devel' scripts/setup-centos8.sh
sed -i '/^function install_gflags.*/i function install_openssl {\n
wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make
install \n cd ..\n}\n' scripts/setup-centos8.sh
sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl'
scripts/setup-centos8.sh
+ sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure
CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh
if [ $ENABLE_HDFS == "ON" ]; then
sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd
"\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n
cmake_install\n}\n' scripts/setup-centos8.sh
sed -i '/^ run_and_time install_fbthrift/a \ run_and_time
install_libhdfs3' scripts/setup-centos8.sh
sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\'
scripts/setup-centos8.sh
fi
- if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then
- sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure
CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh
- fi
+
sed -i "s/yum -y install/sudo yum -y install/"
${VELOX_HOME}/scripts/setup-adapters.sh
if [ $ENABLE_S3 == "ON" ]; then
sed -i '/^ run_and_time install_fbthrift/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos8.sh
@@ -172,15 +163,12 @@ function process_setup_centos7 {
# install gtest
sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_gtest'
scripts/setup-centos7.sh
-
+ sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf'
scripts/setup-centos7.sh
if [ $ENABLE_HDFS = "ON" ]; then
sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd
"\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n
cmake_install\n}\n' scripts/setup-centos7.sh
sed -i '/^ run_and_time install_folly/a \ \ run_and_time
install_libhdfs3' scripts/setup-centos7.sh
sed -i '/^dnf_install ccache/a\ \ yasm \\' scripts/setup-centos7.sh
fi
- if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then
- sed -i '/^ run_and_time install_folly/a \ \ run_and_time
install_protobuf' scripts/setup-centos7.sh
- fi
sed -i "s/yum -y install/sudo yum -y install/"
${VELOX_HOME}/scripts/setup-adapters.sh
if [ $ENABLE_S3 == "ON" ]; then
sed -i '/^ run_and_time install_folly/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos7.sh
@@ -219,7 +207,6 @@ function process_setup_tencentos32 {
echo "Preparing Velox source code..."
echo "ENABLE_HDFS=${ENABLE_HDFS}"
-echo "BUILD_PROTOBUF=${BUILD_PROTOBUF}"
CURRENT_DIR=$(
cd "$(dirname "$BASH_SOURCE")"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
