This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ca49ab9ce [VL] Daily Update Velox Version (2024_07_29) (#6616)
ca49ab9ce is described below
commit ca49ab9ce13e2c96250446ef6f3deb0539eef801
Author: Hongze Zhang <[email protected]>
AuthorDate: Mon Jul 29 16:56:48 2024 +0800
[VL] Daily Update Velox Version (2024_07_29) (#6616)
Co-authored-by: joey.ljy <[email protected]>
---
ep/build-velox/src/get_velox.sh | 28 +++++++--------
ep/build-velox/src/setup-centos7.sh | 18 ++++------
ep/build-velox/src/setup-centos8.sh | 68 +++++++++++--------------------------
3 files changed, 38 insertions(+), 76 deletions(-)
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 841787d64..d9b22482d 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_07_26
+VELOX_BRANCH=2024_07_29
VELOX_HOME=""
OS=`uname -s`
@@ -70,25 +70,22 @@ function process_setup_ubuntu {
ensure_pattern_matched '\${SUDO} apt install -y libunwind-dev'
scripts/setup-ubuntu.sh
sed -i 's/${SUDO} apt install -y libunwind-dev//' scripts/setup-ubuntu.sh
ensure_pattern_matched 'ccache' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ libiberty-dev \\' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ libxml2-dev \\' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ libkrb5-dev \\' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ libgsasl7-dev \\' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ libuuid1 \\' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ uuid-dev \\' scripts/setup-ubuntu.sh
- sed -i '/ccache/a\ curl \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache/a\ libiberty-dev \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache/a\ libxml2-dev \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache/a\ libkrb5-dev \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache/a\ libgsasl7-dev \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache/a\ libuuid1 \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache/a\ uuid-dev \\' scripts/setup-ubuntu.sh
ensure_pattern_matched 'libgmock-dev' scripts/setup-ubuntu.sh
sed -i '/libgmock-dev/d' scripts/setup-ubuntu.sh # resolved by
ep/build-velox/build/velox_ep/CMake/resolve_dependency_modules/gtest.cmake
- ensure_pattern_matched 'github_checkout boostorg\/boost
\"\${BOOST_VERSION}\" --recursive' scripts/setup-ubuntu.sh
- sed -i 's/github_checkout boostorg\/boost \"\${BOOST_VERSION}\"
--recursive/wget_and_untar
https:\/\/github.com\/boostorg\/boost\/releases\/download\/boost-1.84.0\/boost-1.84.0.tar.gz
boost \&\& cd boost/g' scripts/setup-ubuntu.sh
ensure_pattern_matched 'function install_folly' scripts/setup-ubuntu.sh
- sed -i '/^function install_folly.*/i function install_protobuf {\n wget
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n
tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure
CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make
install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh
+ sed -i '/^function install_folly.*/i function install_protobuf {\n
wget_and_untar
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz
protobuf\n (\n cd protobuf\n ./configure CXXFLAGS="-fPIC"
--prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo
ldconfig\n )\n}\n' scripts/setup-ubuntu.sh
ensure_pattern_matched ' run_and_time install_folly' scripts/setup-ubuntu.sh
sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf'
scripts/setup-ubuntu.sh
# Required by lib hdfs.
ensure_pattern_matched 'ccache ' scripts/setup-ubuntu.sh
- sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh
+ sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh
ensure_pattern_matched 'run_and_time install_conda' scripts/setup-ubuntu.sh
sed -i '/run_and_time install_conda/d' scripts/setup-ubuntu.sh
# Just depends on Gluten to install arrow libs since Gluten will apply some
patches to Arrow source and uses different build options.
@@ -107,11 +104,10 @@ function process_setup_centos9 {
ensure_pattern_matched 'dnf_install' scripts/setup-centos9.sh
sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12
git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/'
scripts/setup-centos9.sh
- sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}'
scripts/setup-centos9.sh
- sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel
libuuid-devel' scripts/setup-centos9.sh
+ sed -i '/^.*dnf_install autoconf/a\ dnf_install libxml2-devel
libgsasl-devel libuuid-devel' scripts/setup-centos9.sh
ensure_pattern_matched 'install_gflags' scripts/setup-centos9.sh
- sed -i '/^function install_gflags.*/i function install_openssl {\n
wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make
install \n cd ..\n}\n' scripts/setup-centos9.sh
+ sed -i '/^function install_gflags.*/i function install_openssl {\n
wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl \n cd openssl \n ./config no-shared && make depend && make && sudo
make install \n cd ..\n}\n' scripts/setup-centos9.sh
ensure_pattern_matched 'install_fbthrift' scripts/setup-centos9.sh
sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl'
scripts/setup-centos9.sh
diff --git a/ep/build-velox/src/setup-centos7.sh
b/ep/build-velox/src/setup-centos7.sh
index 6f76112ae..34d7bcfb6 100755
--- a/ep/build-velox/src/setup-centos7.sh
+++ b/ep/build-velox/src/setup-centos7.sh
@@ -74,8 +74,8 @@ function install_ninja {
function install_folly {
cd "${DEPENDENCY_DIR}"
- github_checkout facebook/folly "${FB_OS_VERSION}"
- cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
+ wget_and_untar
https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz
folly
+ cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
}
function install_conda {
@@ -99,22 +99,19 @@ function install_openssl {
function install_gflags {
cd "${DEPENDENCY_DIR}"
wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
- cd gflags
- cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
+ cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
}
function install_glog {
cd "${DEPENDENCY_DIR}"
wget_and_untar https://github.com/google/glog/archive/v0.5.0.tar.gz glog
- cd glog
- cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DCMAKE_INSTALL_PREFIX:PATH=/usr/local
+ cmake_install glog -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DCMAKE_INSTALL_PREFIX:PATH=/usr/local
}
function install_snappy {
cd "${DEPENDENCY_DIR}"
wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
- cd snappy
- cmake_install -DSNAPPY_BUILD_TESTS=OFF
+ cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
}
function install_dwarf {
@@ -213,10 +210,7 @@ function install_duckdb {
if $BUILD_DUCKDB ; then
echo 'Building DuckDB'
wget_and_untar
https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
- (
- cd duckdb
- cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
- )
+ cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
fi
}
diff --git a/ep/build-velox/src/setup-centos8.sh
b/ep/build-velox/src/setup-centos8.sh
index c8078893a..771c2ab83 100755
--- a/ep/build-velox/src/setup-centos8.sh
+++ b/ep/build-velox/src/setup-centos8.sh
@@ -41,7 +41,10 @@ BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
export CC=/opt/rh/gcc-toolset-9/root/bin/gcc
export CXX=/opt/rh/gcc-toolset-9/root/bin/g++
-DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
+FB_OS_VERSION="v2024.05.20.00"
+FMT_VERSION="10.1.1"
+BOOST_VERSION="boost-1.84.0"
+
function dnf_install {
dnf install -y -q --setopt=install_weak_deps=False "$@"
}
@@ -74,27 +77,22 @@ function install_conda {
function install_openssl {
wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl
- cd openssl
- ./config no-shared && make depend && make && sudo make install
- cd ..
+ (
+ cd openssl
+ ./config no-shared && make depend && make && sudo make install
+ )
}
function install_gflags {
# Remove an older version if present.
dnf remove -y gflags
wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
- (
- cd gflags
- cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
- )
+ cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
}
function install_glog {
wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
- (
- cd glog
- cmake_install -DBUILD_SHARED_LIBS=ON
- )
+ cmake_install glog -DBUILD_SHARED_LIBS=ON
}
function install_lzo {
@@ -108,7 +106,7 @@ function install_lzo {
}
function install_boost {
- wget_and_untar
https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz
boost
+ wget_and_untar
https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz
boost
(
cd boost
./bootstrap.sh --prefix=/usr/local
@@ -118,18 +116,12 @@ function install_boost {
function install_snappy {
wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
- (
- cd snappy
- cmake_install -DSNAPPY_BUILD_TESTS=OFF
- )
+ cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
}
function install_fmt {
- wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
- (
- cd fmt
- cmake_install -DFMT_TEST=OFF
- )
+ wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz
fmt
+ cmake_install fmt -DFMT_TEST=OFF
}
function install_protobuf {
@@ -143,56 +135,36 @@ function install_protobuf {
)
}
-FB_OS_VERSION="v2024.05.20.00"
-
function install_fizz {
wget_and_untar
https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz
fizz
- (
- cd fizz/fizz
- cmake_install -DBUILD_TESTS=OFF
- )
+ cmake_install fizz/fizz -DBUILD_TESTS=OFF
}
function install_folly {
wget_and_untar
https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz
folly
- (
- cd folly
- cmake_install -DFOLLY_HAVE_INT128_T=ON
- )
+ cmake_install folly -DFOLLY_HAVE_INT128_T=ON
}
function install_wangle {
wget_and_untar
https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz
wangle
- (
- cd wangle/wangle
- cmake_install -DBUILD_TESTS=OFF
- )
+ cmake_install wangle/wangle -DBUILD_TESTS=OFF
}
function install_fbthrift {
wget_and_untar
https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz
fbthrift
- (
- cd fbthrift
- cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF
-DBUILD_SHARED_LIBS=OFF
- )
+ cmake_install fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF
-DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
}
function install_mvfst {
wget_and_untar
https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz
mvfst
- (
- cd mvfst
- cmake_install -DBUILD_TESTS=OFF
- )
+ cmake_install mvfst -DBUILD_TESTS=OFF
}
function install_duckdb {
if $BUILD_DUCKDB ; then
echo 'Building DuckDB'
wget_and_untar
https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
- (
- cd duckdb
- cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
- )
+ cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
fi
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
