This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new aee02d15b4 [VL] Update setup-rhel.sh (#10935)
aee02d15b4 is described below
commit aee02d15b4a33cac410d3ebd7a656058a795213b
Author: Rong Ma <[email protected]>
AuthorDate: Mon Nov 3 14:52:00 2025 +0000
[VL] Update setup-rhel.sh (#10935)
---
ep/build-velox/src/setup-rhel.sh | 223 +++++++++++----------------------------
1 file changed, 60 insertions(+), 163 deletions(-)
diff --git a/ep/build-velox/src/setup-rhel.sh b/ep/build-velox/src/setup-rhel.sh
index aa2d798fb8..fd781db314 100755
--- a/ep/build-velox/src/setup-rhel.sh
+++ b/ep/build-velox/src/setup-rhel.sh
@@ -14,46 +14,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# This script documents setting up a rhel host for Velox
+# This script documents setting up a RHEL host for Velox
# development. Running it should make you ready to compile.
#
-#
# Environment variables:
# * INSTALL_PREREQUISITES="N": Skip installation of packages for build.
# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts.
# Use "n" to never wipe directories.
#
# You can also run individual functions below by specifying them as arguments:
-# $ scripts/setup-rhel.sh install_googletest install_fmt
+# $ scripts/setup-rhel.sh install_adapters install_gflags
#
set -efx -o pipefail
# Some of the packages must be build with the same compiler flags
# so that some low level types are the same size. Also, disable warnings.
SCRIPTDIR=./scripts
-source $SCRIPTDIR/setup-helper-functions.sh
-NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
+source $SCRIPTDIR/setup-common.sh
export CXXFLAGS=$(get_cxx_flags) # Used by boost.
export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO.
-CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
-VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared
for use in libvelox.so.
-BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
-BUILD_GEOS="${BUILD_GEOS:-true}"
+SUDO="${SUDO:-""}"
+EXTRA_ARROW_OPTIONS=${EXTRA_ARROW_OPTIONS:-""}
USE_CLANG="${USE_CLANG:-false}"
export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
-SUDO="${SUDO:-""}"
-EXTRA_ARROW_OPTIONS=${EXTRA_ARROW_OPTIONS:-""}
-
-FB_OS_VERSION="v2024.07.01.00"
-FMT_VERSION="10.1.1"
-BOOST_VERSION="boost-1.84.0"
-THRIFT_VERSION="v0.16.0"
-# Note: when updating arrow check if thrift needs an update as well.
-ARROW_VERSION="15.0.0"
-STEMMER_VERSION="2.2.0"
-DUCKDB_VERSION="v0.8.1"
FB_ZSTD_VERSION="1.5.6"
DBL_CONVERSION_VERSION="v3.3.0"
SODIUM_VERSION="libsodium-1.0.20-stable"
@@ -62,7 +47,9 @@ DWARF_VERSION="0.11.1"
BISON_VERSION="bison-3.8.2"
RAPIDJSON_VERSION="v1.1.0"
RE2_VERSION="2023-03-01"
-GEOS_VERSION="3.10.7"
+XXHASH_VERSION="0.8.2"
+GOOGLETEST_VERSION="1.11.0"
+C_ARES_VERSION="v1.34.5"
function dnf_install {
dnf install -y -q --setopt=install_weak_deps=False "$@"
@@ -77,11 +64,8 @@ function install_build_prerequisites {
dnf update -y
dnf_install dnf-plugins-core
dnf_install ninja-build cmake gcc-toolset-12 git wget which bzip2
- dnf_install autoconf automake python3-devel pip libtool
- dnf_install libxml2-devel
-
+ dnf_install autoconf automake python3-devel pip libtool
pip install cmake==3.28.3
-
if [[ ${USE_CLANG} != "false" ]]; then
install_clang15
fi
@@ -98,121 +82,19 @@ function install_velox_deps_from_dnf {
function install_gflags {
# Remove an older version if present.
dnf remove -y gflags
- wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+ wget_and_untar
https://github.com/gflags/gflags/archive/${GFLAGS_VERSION}.tar.gz gflags
cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
}
-function install_glog {
- wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
- cmake_install_dir glog -DBUILD_SHARED_LIBS=ON
-}
-
-function install_lzo {
- wget_and_untar
http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
- (
- cd ${DEPENDENCY_DIR}/lzo
- ./configure --prefix=${INSTALL_PREFIX} --enable-shared --disable-static
--docdir=/usr/share/doc/lzo-2.10
- make "-j${NPROC}"
- make install
- )
-}
-
-function install_boost {
- wget_and_untar
https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz
boost
- (
- cd ${DEPENDENCY_DIR}/boost
- if [[ ${USE_CLANG} != "false" ]]; then
- ./bootstrap.sh --prefix=${INSTALL_PREFIX} --with-toolset="clang-15"
- # Switch the compiler from the clang-15 toolset which doesn't exist
(clang-15.jam) to
- # clang of version 15 when toolset clang-15 is used.
- # This reconciles the project-config.jam generation with what the b2
build system allows for customization.
- sed -i 's/using clang-15/using clang : 15/g' project-config.jam
- ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi toolset=clang-15
--without-python
- else
- ./bootstrap.sh --prefix=${INSTALL_PREFIX}
- ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi --without-python
- fi
- )
-}
-
-function install_snappy {
- wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
- cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF
-}
-
-function install_fmt {
- wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz
fmt
- cmake_install_dir fmt -DFMT_TEST=OFF
-}
-
-function install_protobuf {
- wget_and_untar
https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz
protobuf
- (
- cd ${DEPENDENCY_DIR}/protobuf
- ./configure CXXFLAGS="-fPIC" --prefix=${INSTALL_PREFIX}
- make "-j${NPROC}"
- make install
- ldconfig
- )
-}
-
-function install_fizz {
- wget_and_untar
https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz
fizz
- cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF
-}
-
-function install_folly {
- wget_and_untar
https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz
folly
- cmake_install_dir folly -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED"
-DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
-}
-
-function install_wangle {
- wget_and_untar
https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz
wangle
- cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF
-}
-
-function install_fbthrift {
- wget_and_untar
https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz
fbthrift
- cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF
-DBUILD_SHARED_LIBS=OFF
-}
-
-function install_mvfst {
- wget_and_untar
https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz
mvfst
- cmake_install_dir mvfst -DBUILD_TESTS=OFF
-}
-
-function install_duckdb {
- if $BUILD_DUCKDB ; then
- echo 'Building DuckDB'
- wget_and_untar
https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz
duckdb
- cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
- fi
-}
-
-function install_stemmer {
- wget_and_untar
https://snowballstem.org/dist/libstemmer_c-${STEMMER_VERSION}.tar.gz stemmer
- (
- cd ${DEPENDENCY_DIR}/stemmer
- sed -i '/CPPFLAGS=-Iinclude/ s/$/ -fPIC/' Makefile
- make clean && make "-j${NPROC}"
- ${SUDO} cp libstemmer.a ${INSTALL_PREFIX}/lib/
- ${SUDO} cp include/libstemmer.h ${INSTALL_PREFIX}/include/
- )
-}
-
-function install_thrift {
- wget_and_untar
https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz thrift
- (
- cd ${DEPENDENCY_DIR}/thrift
- ./bootstrap.sh
- EXTRA_CXXFLAGS="-O3 -fPIC"
- # Clang will generate warnings and they need to be suppressed, otherwise
the build will fail.
- if [[ ${USE_CLANG} != "false" ]]; then
- EXTRA_CXXFLAGS="-O3 -fPIC -Wno-inconsistent-missing-override
-Wno-unused-but-set-variable"
- fi
- ./configure --prefix=${INSTALL_PREFIX} --enable-tests=no
--enable-tutorial=no --with-boost=${INSTALL_PREFIX}
CXXFLAGS="${EXTRA_CXXFLAGS}" LDFLAGS="-L${INSTALL_PREFIX}/lib"
- make "-j${NPROC}" install
- )
+function install_cuda {
+ # See https://developer.nvidia.com/cuda-downloads
+ dnf config-manager --add-repo
https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
+ local dashed="$(echo $1 | tr '.' '-')"
+ dnf install -y \
+ cuda-compat-$dashed \
+ cuda-driver-devel-$dashed \
+ cuda-minimal-build-$dashed \
+ cuda-nvrtc-devel-$dashed
}
function install_re2 {
@@ -247,6 +129,18 @@ function install_elfutils-libelf {
popd
}
+function install_xxHash {
+ wget_and_untar
https://github.com/Cyan4973/xxHash/archive/refs/tags/v${XXHASH_VERSION}.tar.gz
xxHash
+ cd ${DEPENDENCY_DIR}/xxHash
+ make "-j${NPROC}"
+ make install PREFIX=${INSTALL_PREFIX}
+}
+
+function install_googletest {
+ wget_and_untar
https://github.com/google/googletest/archive/refs/tags/release-${GOOGLETEST_VERSION}.tar.gz
googletest
+ cmake_install_dir googletest -DBUILD_TESTING=OFF
+}
+
function install_double_conversion {
wget_and_untar
https://github.com/google/double-conversion/archive/refs/tags/${DBL_CONVERSION_VERSION}.tar.gz
double-conversion
cmake_install_dir double-conversion -DBUILD_TESTING=OFF
@@ -289,18 +183,6 @@ function install_bison {
)
}
-function install_conda {
- CPU_ARCH=$(uname -m)
- if [[ "$CPU_ARCH" == "amd64" ]]; then
- CPU_ARCH="x86_64"
- fi
- mkdir -p ${DEPENDENCY_DIR}/miniconda3
- curl
https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${CPU_ARCH}.sh -o
${DEPENDENCY_DIR}/miniconda3/miniconda.sh
- bash ${DEPENDENCY_DIR}/miniconda3/miniconda.sh -b -u -p ~/miniconda3
- source ~/miniconda3/bin/activate
- conda init --all
-}
-
function install_rapidjson {
wget_and_untar
https://github.com/Tencent/rapidjson/archive/refs/tags/${RAPIDJSON_VERSION}.tar.gz
rapidjson
(
@@ -312,27 +194,40 @@ function install_rapidjson {
}
function install_c-ares {
- github_checkout c-ares/c-ares v1.34 --depth 1
- cmake_install -DCMAKE_BUILD_TYPE=Release
+ wget_and_untar
https://github.com/c-ares/c-ares/archive/refs/tags/${C_ARES_VERSION}.tar.gz
c-ares
+ cmake_install_dir c-ares -DCMAKE_BUILD_TYPE=Release
}
-function install_cuda {
- # See https://developer.nvidia.com/cuda-downloads
- dnf config-manager --add-repo
https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
- local dashed="$(echo $1 | tr '.' '-')"
- dnf install -y cuda-nvcc-$dashed cuda-cudart-devel-$dashed
cuda-nvrtc-devel-$dashed cuda-driver-devel-$dashed
+function install_s3 {
+ install_aws_deps
+
+ local MINIO_OS="linux"
+ install_minio ${MINIO_OS}
}
-function install_geos {
- if [[ "$BUILD_GEOS" == "true" ]]; then
- wget_and_untar
https://github.com/libgeos/geos/archive/${GEOS_VERSION}.tar.gz geos
- cmake_install_dir geos -DBUILD_TESTING=OFF
- fi
+function install_gcs {
+ # Dependencies of GCS, probably a workaround until the docker image is
rebuilt
+ dnf -y install npm curl-devel
+ install_c-ares
+ install_gcs-sdk-cpp
+}
+
+function install_abfs {
+ # Dependencies of Azure Storage Blob cpp
+ dnf -y install perl-IPC-Cmd openssl libxml2-devel
+ install_azure-storage-sdk-cpp
+}
+
+function install_adapters {
+ run_and_time install_s3
+ run_and_time install_gcs
+ run_and_time install_abfs
}
function install_velox_deps {
run_and_time install_velox_deps_from_dnf
- run_and_time install_conda
+ run_and_time install_xxHash
+ run_and_time install_googletest
run_and_time install_re2
run_and_time install_double_conversion
run_and_time install_libdwarf
@@ -345,11 +240,11 @@ function install_velox_deps {
run_and_time install_c-ares
run_and_time install_gflags
run_and_time install_glog
- run_and_time install_lzo
run_and_time install_snappy
run_and_time install_boost
run_and_time install_protobuf
run_and_time install_fmt
+ run_and_time install_fast_float
run_and_time install_folly
run_and_time install_fizz
run_and_time install_wangle
@@ -358,6 +253,7 @@ function install_velox_deps {
run_and_time install_duckdb
run_and_time install_stemmer
run_and_time install_thrift
+ run_and_time install_simdjson
run_and_time install_geos
}
@@ -403,3 +299,4 @@ function install_velox_deps {
dnf clean all
fi
)
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
