This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d60d4458b [VL] Move setup-centos7.sh & setup-centos8.sh into Gluten
and clean up some script code (#6559)
d60d4458b is described below
commit d60d4458b9d38d0eff565266cb1c2bd02510d4a9
Author: PHILO-HE <[email protected]>
AuthorDate: Thu Jul 25 11:08:21 2024 +0800
[VL] Move setup-centos7.sh & setup-centos8.sh into Gluten and clean up some
script code (#6559)
---
dev/build_helper_functions.sh | 42 +++++-
dev/builddeps-veloxbe.sh | 28 +++-
ep/build-velox/src/get_velox.sh | 191 ++++--------------------
ep/build-velox/src/setup-centos7.sh | 283 ++++++++++++++++++++++++++++++++++++
ep/build-velox/src/setup-centos8.sh | 250 +++++++++++++++++++++++++++++++
5 files changed, 623 insertions(+), 171 deletions(-)
diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh
index 92ae5c29a..b69a36c29 100644
--- a/dev/build_helper_functions.sh
+++ b/dev/build_helper_functions.sh
@@ -81,6 +81,28 @@ function get_cxx_flags {
}
+function github_checkout {
+ local REPO=$1
+ shift
+ local VERSION=$1
+ shift
+ local GIT_CLONE_PARAMS=$@
+ local DIRNAME=$(basename $REPO)
+ SUDO="${SUDO:-""}"
+ cd "${DEPENDENCY_DIR}"
+ if [ -z "${DIRNAME}" ]; then
+ echo "Failed to get repo name from ${REPO}"
+ exit 1
+ fi
+ if [ -d "${DIRNAME}" ] && prompt "${DIRNAME} already exists. Delete?"; then
+ ${SUDO} rm -rf "${DIRNAME}"
+ fi
+ if [ ! -d "${DIRNAME}" ]; then
+ git clone -q -b $VERSION $GIT_CLONE_PARAMS "https://github.com/${REPO}.git";
+ fi
+ cd "${DIRNAME}"
+}
+
function wget_and_untar {
local URL=$1
local DIR=$2
@@ -130,15 +152,17 @@ function setup_macos {
function setup_linux {
local LINUX_DISTRIBUTION=$(. /etc/os-release && echo ${ID})
local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID})
+ CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+ GLUTEN_VELOX_SCRIPT_HOME=$CURRENT_DIR/../ep/build-velox/src
if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian"
|| "$LINUX_DISTRIBUTION" == "pop" ]]; then
scripts/setup-ubuntu.sh
elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then
case "$LINUX_VERSION_ID" in
9) scripts/setup-centos9.sh ;;
- 8) scripts/setup-centos8.sh ;;
+ 8) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
7)
- scripts/setup-centos7.sh
+ $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
set +u
export
PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
source /opt/rh/devtoolset-9/enable
@@ -152,13 +176,13 @@ function setup_linux {
elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then
case "${LINUX_VERSION_ID:0:1}" in
2)
- scripts/setup-centos7.sh
+ $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
set +u
export
PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
source /opt/rh/devtoolset-9/enable
set -u
;;
- 3) scripts/setup-centos8.sh ;;
+ 3) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
*)
echo "Unsupported alinux version: $LINUX_VERSION_ID"
exit 1
@@ -167,13 +191,13 @@ function setup_linux {
elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then
case "$LINUX_VERSION_ID" in
2.4)
- scripts/setup-centos7.sh
+ $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
set +u
export
PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
source /opt/rh/devtoolset-9/enable
set -u
;;
- 3.2) scripts/setup-centos8.sh ;;
+ 3.2) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
*)
echo "Unsupported tencentos version: $LINUX_VERSION_ID"
exit 1
@@ -184,3 +208,9 @@ function setup_linux {
exit 1
fi
}
+
+function install_libhdfs3 {
+ github_checkout oap-project/libhdfs3 master
+ cmake_install
+}
+
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 38f3978ef..82c9cfc8d 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -223,7 +223,7 @@ function build_velox_backend {
(
cd $GLUTEN_DIR/ep/build-velox/src
- ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --enable_s3=$ENABLE_S3
--enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER
+ ./get_velox.sh $VELOX_PARAMETER
)
if [ "$VELOX_HOME" == "" ]; then
@@ -232,6 +232,9 @@ fi
OS=`uname -s`
ARCH=`uname -m`
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-$CURRENT_DIR/../ep/_ep}
+mkdir -p ${DEPENDENCY_DIR}
+
source $GLUTEN_DIR/dev/build_helper_functions.sh
if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
echo "Start to install dependencies"
@@ -244,6 +247,29 @@ if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [
$RUN_SETUP_SCRIPT == "ON" ]; then
echo "Unsupported kernel: $OS"
exit 1
fi
+ if [ $ENABLE_S3 == "ON" ]; then
+ if [ $OS == 'Darwin' ]; then
+ echo "S3 is not supported on MacOS."
+ exit 1
+ fi
+ ${VELOX_HOME}/scripts/setup-adapters.sh aws
+ fi
+ if [ $ENABLE_HDFS == "ON" ]; then
+ if [ $OS == 'Darwin' ]; then
+ echo "HDFS is not supported on MacOS."
+ exit 1
+ fi
+ pushd $VELOX_HOME
+ install_libhdfs3
+ popd
+ fi
+ if [ $ENABLE_GCS == "ON" ]; then
+ ${VELOX_HOME}/scripts/setup-adapters.sh gcs
+ fi
+ if [ $ENABLE_ABFS == "ON" ]; then
+ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON
+ ${VELOX_HOME}/scripts/setup-adapters.sh abfs
+ fi
popd
fi
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 30e13926b..164bf88a4 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -16,20 +16,10 @@
set -exu
-
VELOX_REPO=https://github.com/oap-project/velox.git
VELOX_BRANCH=2024_07_24
VELOX_HOME=""
-#Set on run gluten on HDFS
-ENABLE_HDFS=OFF
-#Set on run gluten on S3
-ENABLE_S3=OFF
-#Set on run gluten on GCS
-ENABLE_GCS=OFF
-#Set on run gluten on ABFS
-ENABLE_ABFS=OFF
-
OS=`uname -s`
for arg in "$@"; do
@@ -46,22 +36,6 @@ for arg in "$@"; do
VELOX_HOME=("${arg#*=}")
shift # Remove argument name from processing
;;
- --enable_hdfs=*)
- ENABLE_HDFS=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
- --enable_s3=*)
- ENABLE_S3=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
- --enable_gcs=*)
- ENABLE_GCS=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
- --enable_abfs=*)
- ENABLE_ABFS=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
*)
OTHER_ARGUMENTS+=("$1")
shift # Remove generic argument from processing
@@ -112,24 +86,9 @@ function process_setup_ubuntu {
sed -i '/^function install_folly.*/i function install_protobuf {\n wget
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n
tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure
CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make
install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh
ensure_pattern_matched ' run_and_time install_folly' scripts/setup-ubuntu.sh
sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf'
scripts/setup-ubuntu.sh
- if [ $ENABLE_HDFS == "ON" ]; then
- sed -i '/^function install_folly.*/i function install_libhdfs3 {\n
github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n'
scripts/setup-ubuntu.sh
- ensure_pattern_matched ' run_and_time install_protobuf'
scripts/setup-ubuntu.sh
- sed -i '/^ run_and_time install_protobuf/a \ \ run_and_time
install_libhdfs3' scripts/setup-ubuntu.sh
- ensure_pattern_matched 'ccache ' scripts/setup-ubuntu.sh
- sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh
- fi
- ensure_pattern_matched 'apt install -y' scripts/setup-adapters.sh
- sed -i "s/apt install -y/sudo apt install -y/" scripts/setup-adapters.sh
- if [ $ENABLE_S3 == "ON" ]; then
- sed -i '/^ run_and_time install_folly/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh
- fi
- if [ $ENABLE_GCS == "ON" ]; then
- sed -i '/^ run_and_time install_folly/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-ubuntu.sh
- fi
- if [ $ENABLE_ABFS == "ON" ]; then
- sed -i '/^ run_and_time install_folly/a \ \ export
AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh
abfs' scripts/setup-ubuntu.sh
- fi
+ # Required by lib hdfs.
+ ensure_pattern_matched 'ccache ' scripts/setup-ubuntu.sh
+ sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh
ensure_pattern_matched 'run_and_time install_conda' scripts/setup-ubuntu.sh
sed -i '/run_and_time install_conda/d' scripts/setup-ubuntu.sh
# Just depends on Gluten to install arrow libs since Gluten will apply some
patches to Arrow source and uses different build options.
@@ -158,113 +117,28 @@ function process_setup_centos9 {
sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl'
scripts/setup-centos9.sh
sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure
CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh
- if [ $ENABLE_HDFS == "ON" ]; then
- sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd
"\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n
cmake_install\n}\n' scripts/setup-centos9.sh
- sed -i '/^ run_and_time install_fbthrift/a \ run_and_time
install_libhdfs3' scripts/setup-centos9.sh
- sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\'
scripts/setup-centos9.sh
- fi
- sed -i "s/yum -y install/sudo yum -y install/"
${VELOX_HOME}/scripts/setup-adapters.sh
- if [ $ENABLE_S3 == "ON" ]; then
- sed -i '/^ run_and_time install_fbthrift/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos9.sh
- fi
- if [ $ENABLE_GCS == "ON" ]; then
- sed -i '/^ run_and_time install_fbthrift/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos9.sh
- fi
- if [ $ENABLE_ABFS == "ON" ]; then
- sed -i '/^ run_and_time install_fbthrift/a \ \ export
AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh
abfs' scripts/setup-centos9.sh
- fi
-}
-
-
-function process_setup_centos8 {
- # Allows other version of git already installed.
- if [ -z "$(which git)" ]; then
- dnf install -y -q --setopt=install_weak_deps=False git
- fi
- # make this function Reentrant
- git checkout scripts/setup-centos8.sh
- # need set BUILD_SHARED_LIBS flag for thrift
- sed -i "/cd fbthrift/{n;s/cmake_install -Denable_tests=OFF/cmake_install
-Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF/;}" scripts/setup-centos8.sh
- # No need to re-install git.
- sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-9
git/dnf_install ninja-build cmake curl ccache gcc-toolset-9/'
scripts/setup-centos8.sh
- sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}'
scripts/setup-centos8.sh
- sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel
libuuid-devel' scripts/setup-centos8.sh
- sed -i '/^function install_gflags.*/i function install_openssl {\n
wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make
install \n cd ..\n}\n' scripts/setup-centos8.sh
- sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl'
scripts/setup-centos8.sh
- sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure
CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh
-
- if [ $ENABLE_HDFS == "ON" ]; then
- sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd
"\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n
cmake_install\n}\n' scripts/setup-centos8.sh
- sed -i '/^ run_and_time install_fbthrift/a \ run_and_time
install_libhdfs3' scripts/setup-centos8.sh
- sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\'
scripts/setup-centos8.sh
- fi
-
- sed -i "s/yum -y install/sudo yum -y install/"
${VELOX_HOME}/scripts/setup-adapters.sh
- if [ $ENABLE_S3 == "ON" ]; then
- sed -i '/^ run_and_time install_fbthrift/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos8.sh
- fi
- if [ $ENABLE_GCS == "ON" ]; then
- sed -i '/^ run_and_time install_fbthrift/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos8.sh
- fi
- if [ $ENABLE_ABFS == "ON" ]; then
- sed -i '/^ run_and_time install_fbthrift/a \ \ export
AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh
abfs' scripts/setup-centos8.sh
- fi
-}
-
-function process_setup_centos7 {
- # Allows other version of git already installed.
- if [ -z "$(which git)" ]; then
- dnf install -y -q --setopt=install_weak_deps=False git
- fi
- # make this function Reentrant
- git checkout scripts/setup-centos7.sh
-
- # No need to re-install git.
- sed -i 's/dnf_install ccache git/dnf_install ccache/'
scripts/setup-centos7.sh
-
- # install gtest
- sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_gtest'
scripts/setup-centos7.sh
- sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf'
scripts/setup-centos7.sh
- sed -i
's/https:\/\/cmake.org\/files\/v3.25\/cmake-3.25.1.tar.gz/https:\/\/cmake.org\/files\/v3.28\/cmake-3.28.3.tar.gz/'
scripts/setup-centos7.sh
- if [ $ENABLE_HDFS = "ON" ]; then
- sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd
"\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n
cmake_install\n}\n' scripts/setup-centos7.sh
- sed -i '/^ run_and_time install_protobuf/a \ \ run_and_time
install_libhdfs3' scripts/setup-centos7.sh
- sed -i '/^dnf_install ccache/a\ \ yasm \\' scripts/setup-centos7.sh
- fi
- sed -i "s/yum -y install/sudo yum -y install/"
${VELOX_HOME}/scripts/setup-adapters.sh
- if [ $ENABLE_S3 == "ON" ]; then
- sed -i '/^ run_and_time install_folly/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos7.sh
- fi
- if [ $ENABLE_GCS == "ON" ]; then
- sed -i '/^ run_and_time install_folly/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos7.sh
- fi
- if [ $ENABLE_ABFS == "ON" ]; then
- sed -i '/^function cmake_install_deps.*/i function install_curl {\n cd
"${DEPENDENCY_DIR}" \n github_checkout curl/curl curl-7_68_0 \n cmake
-DCMAKE_BUILD_TYPE=Release -DOPENSSL_USE_STATIC_LIBS=TRUE
-DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON && cmake --build .
&& sudo cmake --install . \n}\n' scripts/setup-centos7.sh
- sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_curl'
scripts/setup-centos7.sh
- sed -i '/^ run_and_time install_curl/a \ \ export
AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh
abfs' scripts/setup-centos7.sh
- fi
+ # Required by lib hdfs.
+ ensure_pattern_matched 'dnf_install ninja-build' scripts/setup-centos9.sh
+ sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\'
scripts/setup-centos9.sh
}
function process_setup_alinux3 {
- process_setup_centos8
- sed -i "s/.*dnf_install epel-release/#&/" scripts/setup-centos8.sh
- sed -i "s/.*run_and_time install_conda/#&/" scripts/setup-centos8.sh
- sed -i "s/.*dnf config-manager --set-enabled powertools/#&/"
scripts/setup-centos8.sh
- sed -i "s/gcc-toolset-9 //" scripts/setup-centos8.sh
- sed -i "s/.*source \/opt\/rh\/gcc-toolset-9\/enable/#&/"
scripts/setup-centos8.sh
- sed -i 's|^export CC=/opt/rh/gcc-toolset-9/root/bin/gcc|# &|'
scripts/setup-centos8.sh
- sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|'
scripts/setup-centos8.sh
- sed -i 's/python39 python39-devel python39-pip //g' scripts/setup-centos8.sh
- sed -i "s/.*pip.* install/#&/" scripts/setup-centos8.sh
+ sed -i "s/.*dnf_install epel-release/#&/" ${CURRENT_DIR}/setup-centos8.sh
+ sed -i "s/.*run_and_time install_conda/#&/" ${CURRENT_DIR}/setup-centos8.sh
+ sed -i "s/.*dnf config-manager --set-enabled powertools/#&/"
${CURRENT_DIR}/setup-centos8.sh
+ sed -i "s/gcc-toolset-9 //" ${CURRENT_DIR}/setup-centos8.sh
+ sed -i "s/.*source \/opt\/rh\/gcc-toolset-9\/enable/#&/"
${CURRENT_DIR}/setup-centos8.sh
+ sed -i 's|^export CC=/opt/rh/gcc-toolset-9/root/bin/gcc|# &|'
${CURRENT_DIR}/setup-centos8.sh
+ sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|'
${CURRENT_DIR}/setup-centos8.sh
+ sed -i 's/python39 python39-devel python39-pip //g'
${CURRENT_DIR}/setup-centos8.sh
+ sed -i "s/.*pip.* install/#&/" ${CURRENT_DIR}/setup-centos8.sh
}
function process_setup_tencentos32 {
- process_setup_centos8
- sed -i "s/.*dnf config-manager --set-enabled powertools/#&/"
scripts/setup-centos8.sh
+ sed -i "s/.*dnf config-manager --set-enabled powertools/#&/"
${CURRENT_DIR}/setup-centos8.sh
}
echo "Preparing Velox source code..."
-echo "ENABLE_HDFS=${ENABLE_HDFS}"
CURRENT_DIR=$(
cd "$(dirname "$BASH_SOURCE")"
@@ -327,44 +201,33 @@ function setup_linux {
elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then
case "$LINUX_VERSION_ID" in
9) process_setup_centos9 ;;
- 8) process_setup_centos8 ;;
- 7) process_setup_centos7 ;;
+ 8) ;;
+ 7) ;;
*)
- echo "Unsupport centos version: $LINUX_VERSION_ID"
+ echo "Unsupported centos version: $LINUX_VERSION_ID"
exit 1
;;
esac
elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then
case "${LINUX_VERSION_ID:0:1}" in
- 2) process_setup_centos7 ;;
+ 2) ;;
3) process_setup_alinux3 ;;
*)
- echo "Unsupport alinux version: $LINUX_VERSION_ID"
+ echo "Unsupported alinux version: $LINUX_VERSION_ID"
exit 1
;;
esac
elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then
case "$LINUX_VERSION_ID" in
- 2.4) process_setup_centos7 ;;
+ 2.4) ;;
3.2) process_setup_tencentos32 ;;
*)
- echo "Unsupport tencentos version: $LINUX_VERSION_ID"
+ echo "Unsupported tencentos version: $LINUX_VERSION_ID"
exit 1
;;
esac
else
- echo "Unsupport linux distribution: $LINUX_DISTRIBUTION"
- exit 1
- fi
-}
-
-function setup_macos {
- if [ $ENABLE_HDFS == "ON" ]; then
- echo "Unsupport hdfs"
- exit 1
- fi
- if [ $ENABLE_S3 == "ON" ]; then
- echo "Unsupport s3"
+ echo "Unsupported linux distribution: $LINUX_DISTRIBUTION"
exit 1
fi
}
@@ -372,12 +235,12 @@ function setup_macos {
if [ $OS == 'Linux' ]; then
setup_linux
elif [ $OS == 'Darwin' ]; then
- setup_macos
+ :
else
- echo "Unsupport kernel: $OS"
+ echo "Unsupported kernel: $OS"
exit 1
fi
apply_compilation_fixes $CURRENT_DIR $VELOX_SOURCE_DIR
-echo "Velox-get finished."
+echo "Finished getting Velox code"
diff --git a/ep/build-velox/src/setup-centos7.sh
b/ep/build-velox/src/setup-centos7.sh
new file mode 100755
index 000000000..6f76112ae
--- /dev/null
+++ b/ep/build-velox/src/setup-centos7.sh
@@ -0,0 +1,283 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -efx -o pipefail
+# Some of the packages must be build with the same compiler flags
+# so that some low level types are the same size. Also, disable warnings.
+SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")/../build/velox_ep/scripts
+source $SCRIPTDIR/setup-helper-functions.sh
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-/tmp/velox-deps}
+CPU_TARGET="${CPU_TARGET:-avx}"
+NPROC=$(getconf _NPROCESSORS_ONLN)
+FMT_VERSION=10.1.1
+BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
+export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO.
+export CXXFLAGS=$CFLAGS # Used by boost.
+export CPPFLAGS=$CFLAGS # Used by LZO.
+export
PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
+FB_OS_VERSION=v2024.02.26.00
+
+# shellcheck disable=SC2037
+SUDO="sudo -E"
+
+function run_and_time {
+ time "$@"
+ { echo "+ Finished running $*"; } 2> /dev/null
+}
+
+function dnf_install {
+ $SUDO dnf install -y -q --setopt=install_weak_deps=False "$@"
+}
+
+function yum_install {
+ $SUDO yum install -y "$@"
+}
+
+function wget_and_untar {
+ local URL=$1
+ local DIR=$2
+ mkdir -p "${DIR}"
+ wget -q --max-redirect 3 -O - "${URL}" | tar -xz -C "${DIR}"
--strip-components=1
+}
+
+function install_cmake {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar https://cmake.org/files/v3.28/cmake-3.28.3.tar.gz cmake-3
+ cd cmake-3
+ ./bootstrap --prefix=/usr/local
+ make -j$(nproc)
+ $SUDO make install
+ cmake --version
+}
+
+function install_ninja {
+ cd "${DEPENDENCY_DIR}"
+ github_checkout ninja-build/ninja v1.11.1
+ ./configure.py --bootstrap
+ cmake -Bbuild-cmake
+ cmake --build build-cmake
+ $SUDO cp ninja /usr/local/bin/
+}
+
+function install_folly {
+ cd "${DEPENDENCY_DIR}"
+ github_checkout facebook/folly "${FB_OS_VERSION}"
+ cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
+}
+
+function install_conda {
+ cd "${DEPENDENCY_DIR}"
+ mkdir -p conda && cd conda
+ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+ MINICONDA_PATH=/opt/miniconda-for-velox
+ bash Miniconda3-latest-Linux-x86_64.sh -b -u $MINICONDA_PATH
+}
+
+function install_openssl {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl
+ cd openssl
+ ./config no-shared
+ make depend
+ make
+ $SUDO make install
+}
+
+function install_gflags {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+ cd gflags
+ cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
+}
+
+function install_glog {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar https://github.com/google/glog/archive/v0.5.0.tar.gz glog
+ cd glog
+ cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DCMAKE_INSTALL_PREFIX:PATH=/usr/local
+}
+
+function install_snappy {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
+ cd snappy
+ cmake_install -DSNAPPY_BUILD_TESTS=OFF
+}
+
+function install_dwarf {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar
https://github.com/davea42/libdwarf-code/archive/refs/tags/20210528.tar.gz dwarf
+ cd dwarf
+ #local
URL=https://github.com/davea42/libdwarf-code/releases/download/v0.5.0/libdwarf-0.5.0.tar.xz
+ #local DIR=dwarf
+ #mkdir -p "${DIR}"
+ #wget -q --max-redirect 3 "${URL}"
+ #tar -xf libdwarf-0.5.0.tar.xz -C "${DIR}"
+ #cd dwarf/libdwarf-0.5.0
+ ./configure --enable-shared=no
+ make
+ make check
+ $SUDO make install
+}
+
+function install_re2 {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar
https://github.com/google/re2/archive/refs/tags/2023-03-01.tar.gz re2
+ cd re2
+ $SUDO make install
+}
+
+function install_flex {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar
https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz flex
+ cd flex
+ ./autogen.sh
+ ./configure
+ $SUDO make install
+}
+
+function install_lzo {
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar
http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
+ cd lzo
+ ./configure --prefix=/usr/local --enable-shared --disable-static
--docdir=/usr/local/share/doc/lzo-2.10
+ make "-j$(nproc)"
+ $SUDO make install
+}
+
+function install_boost {
+ # Remove old version.
+ sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_*
/opt/rh/devtoolset-9/root/usr/lib64/dyninst/libboost_*
+ sudo rm -rf /tmp/velox-deps/boost/ /usr/local/include/boost/
/usr/local/lib/cmake/Boost-1.72.0/
+ cd "${DEPENDENCY_DIR}"
+ wget_and_untar
https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz
boost
+ cd boost
+ ./bootstrap.sh --prefix=/usr/local --with-python=/usr/bin/python3
--with-python-root=/usr/lib/python3.6 --without-libraries=python
+ $SUDO ./b2 "-j$(nproc)" -d0 install threading=multi
+}
+
+function install_libhdfs3 {
+ cd "${DEPENDENCY_DIR}"
+ github_checkout apache/hawq master
+ cd depends/libhdfs3
+ sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt
+ sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake
+ sed -i "s/dfs.domain.socket.path\", \"\"/dfs.domain.socket.path\",
\"\/var\/lib\/hadoop-hdfs\/dn_socket\"/g" src/common/SessionConfig.cpp
+ sed -i "s/pos < endOfCurBlock/pos \< endOfCurBlock \&\& pos \- cursor \<\=
128 \* 1024/g" src/client/InputStreamImpl.cpp
+ cmake_install
+}
+
+function install_protobuf {
+ cd "${DEPENDENCY_DIR}"
+ wget
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz
+ tar -xzf protobuf-all-21.4.tar.gz
+ cd protobuf-21.4
+ ./configure CXXFLAGS="-fPIC" --prefix=/usr/local
+ make "-j$(nproc)"
+ $SUDO make install
+}
+
+function install_gtest {
+ cd "${DEPENDENCY_DIR}"
+ wget
https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz
+ tar -xzf release-1.12.1.tar.gz
+ cd googletest-release-1.12.1
+ mkdir -p build && cd build && cmake -DBUILD_GTEST=ON -DBUILD_GMOCK=ON
-DINSTALL_GTEST=ON -DINSTALL_GMOCK=ON -DBUILD_SHARED_LIBS=ON ..
+ make "-j$(nproc)"
+ $SUDO make install
+}
+
+function install_fmt {
+ rm -rf /usr/local/lib64/libfmt.a
+ rm -rf /usr/local/lib64/cmake/fmt
+ rm -rf /usr/local/include/fmt
+ rm -rf fmt
+ wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
+ cmake_install fmt -DFMT_TEST=OFF
+}
+
+function install_duckdb {
+ if $BUILD_DUCKDB ; then
+ echo 'Building DuckDB'
+ wget_and_untar
https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
+ (
+ cd duckdb
+ cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
+ )
+ fi
+}
+
+function install_prerequisites {
+ run_and_time install_lzo
+ run_and_time install_boost
+ run_and_time install_re2
+ run_and_time install_flex
+ run_and_time install_openssl
+ run_and_time install_gflags
+ run_and_time install_glog
+ run_and_time install_snappy
+ run_and_time install_dwarf
+}
+
+function install_velox_deps {
+ run_and_time install_fmt
+ run_and_time install_folly
+ run_and_time install_protobuf
+ run_and_time install_gtest
+ run_and_time install_conda
+ run_and_time install_duckdb
+}
+
+$SUDO dnf makecache
+
+# dnf install dependency libraries
+dnf_install epel-release dnf-plugins-core # For ccache, ninja
+# PowerTools only works on CentOS8
+# dnf config-manager --set-enabled powertools
+dnf_install ccache wget which libevent-devel \
+ yasm \
+ openssl-devel libzstd-devel lz4-devel double-conversion-devel \
+ curl-devel libxml2-devel libgsasl-devel libuuid-devel patch
+
+$SUDO dnf remove -y gflags
+
+# Required for Thrift
+dnf_install autoconf automake libtool bison python3 python3-devel
+
+# Required for build flex
+dnf_install gettext-devel texinfo help2man
+
+# dnf_install conda
+
+# Activate gcc9; enable errors on unset variables afterwards.
+# GCC9 install via yum and devtoolset
+# dnf install gcc-toolset-9 only works on CentOS8
+
+$SUDO yum makecache
+yum_install centos-release-scl
+yum_install devtoolset-9
+source /opt/rh/devtoolset-9/enable || exit 1
+gcc --version
+set -u
+
+# Build from source
+[ -d "$DEPENDENCY_DIR" ] || mkdir -p "$DEPENDENCY_DIR"
+
+run_and_time install_cmake
+run_and_time install_ninja
+
+install_prerequisites
+install_velox_deps
diff --git a/ep/build-velox/src/setup-centos8.sh
b/ep/build-velox/src/setup-centos8.sh
new file mode 100755
index 000000000..c8078893a
--- /dev/null
+++ b/ep/build-velox/src/setup-centos8.sh
@@ -0,0 +1,250 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script documents setting up a Centos8 host for Velox
+# development. Running it should make you ready to compile.
+#
+# Environment variables:
+# * INSTALL_PREREQUISITES="N": Skip installation of packages for build.
+# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts.
+# Use "n" to never wipe directories.
+#
+# You can also run individual functions below by specifying them as arguments:
+# $ ./setup-centos8.sh install_googletest install_fmt
+#
+
+set -efx -o pipefail
+# Some of the packages must be build with the same compiler flags
+# so that some low level types are the same size. Also, disable warnings.
+SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")/../build/velox_ep/scripts
+source $SCRIPTDIR/setup-helper-functions.sh
+CPU_TARGET="${CPU_TARGET:-avx}"
+NPROC=$(getconf _NPROCESSORS_ONLN)
+export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO.
+export CXXFLAGS=$CFLAGS # Used by boost.
+export CPPFLAGS=$CFLAGS # Used by LZO.
+CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
+BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
+export CC=/opt/rh/gcc-toolset-9/root/bin/gcc
+export CXX=/opt/rh/gcc-toolset-9/root/bin/g++
+
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
+function dnf_install {
+ dnf install -y -q --setopt=install_weak_deps=False "$@"
+}
+
+# Install packages required for build.
+function install_build_prerequisites {
+ dnf update -y
+ dnf_install epel-release dnf-plugins-core # For ccache, ninja
+ dnf config-manager --set-enabled powertools
+ dnf update -y
+ dnf_install ninja-build curl ccache gcc-toolset-9 git wget which
+ dnf_install yasm
+ dnf_install autoconf automake python39 python39-devel python39-pip libtool
+ pip3.9 install cmake==3.28.3
+}
+
+# Install dependencies from the package managers.
+function install_velox_deps_from_dnf {
+ dnf_install libevent-devel \
+ openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
+ libdwarf-devel curl-devel libicu-devel bison flex libsodium-devel
+
+ # install sphinx for doc gen
+ pip3.9 install sphinx sphinx-tabs breathe sphinx_rtd_theme
+}
+
+function install_conda {
+ dnf_install conda
+}
+
+function install_openssl {
+ wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl
+ cd openssl
+ ./config no-shared && make depend && make && sudo make install
+ cd ..
+}
+
+function install_gflags {
+ # Remove an older version if present.
+ dnf remove -y gflags
+ wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+ (
+ cd gflags
+ cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
+ )
+}
+
+function install_glog {
+ wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
+ (
+ cd glog
+ cmake_install -DBUILD_SHARED_LIBS=ON
+ )
+}
+
+function install_lzo {
+ wget_and_untar
http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
+ (
+ cd lzo
+ ./configure --prefix=/usr --enable-shared --disable-static
--docdir=/usr/share/doc/lzo-2.10
+ make "-j$(nproc)"
+ make install
+ )
+}
+
+function install_boost {
+ wget_and_untar
https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz
boost
+ (
+ cd boost
+ ./bootstrap.sh --prefix=/usr/local
+ ./b2 "-j$(nproc)" -d0 install threading=multi --without-python
+ )
+}
+
+function install_snappy {
+ wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
+ (
+ cd snappy
+ cmake_install -DSNAPPY_BUILD_TESTS=OFF
+ )
+}
+
+function install_fmt {
+ wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
+ (
+ cd fmt
+ cmake_install -DFMT_TEST=OFF
+ )
+}
+
+function install_protobuf {
+ wget_and_untar
https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz
protobuf
+ (
+ cd protobuf
+ ./configure CXXFLAGS="-fPIC" --prefix=/usr/local
+ make "-j${NPROC}"
+ make install
+ ldconfig
+ )
+}
+
+FB_OS_VERSION="v2024.05.20.00"
+
+function install_fizz {
+ wget_and_untar
https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz
fizz
+ (
+ cd fizz/fizz
+ cmake_install -DBUILD_TESTS=OFF
+ )
+}
+
+function install_folly {
+ wget_and_untar
https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz
folly
+ (
+ cd folly
+ cmake_install -DFOLLY_HAVE_INT128_T=ON
+ )
+}
+
+function install_wangle {
+ wget_and_untar
https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz
wangle
+ (
+ cd wangle/wangle
+ cmake_install -DBUILD_TESTS=OFF
+ )
+}
+
+function install_fbthrift {
+ wget_and_untar
https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz
fbthrift
+ (
+ cd fbthrift
+ cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF
-DBUILD_SHARED_LIBS=OFF
+ )
+}
+
+function install_mvfst {
+ wget_and_untar
https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz
mvfst
+ (
+ cd mvfst
+ cmake_install -DBUILD_TESTS=OFF
+ )
+}
+
+function install_duckdb {
+ if $BUILD_DUCKDB ; then
+ echo 'Building DuckDB'
+ wget_and_untar
https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
+ (
+ cd duckdb
+ cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
+ )
+ fi
+}
+
+function install_cuda {
+ # See https://developer.nvidia.com/cuda-downloads
+ dnf config-manager --add-repo
https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo
+ yum install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1
| tr '.' '-')
+}
+
+function install_velox_deps {
+ run_and_time install_velox_deps_from_dnf
+ run_and_time install_conda
+ run_and_time install_gflags
+ run_and_time install_glog
+ run_and_time install_lzo
+ run_and_time install_snappy
+ run_and_time install_boost
+ run_and_time install_protobuf
+ run_and_time install_fmt
+ run_and_time install_folly
+ run_and_time install_fizz
+ run_and_time install_wangle
+ run_and_time install_mvfst
+ run_and_time install_fbthrift
+ run_and_time install_openssl
+ run_and_time install_duckdb
+}
+
+(return 2> /dev/null) && return # If script was sourced, don't run commands.
+
+(
+ if [[ $# -ne 0 ]]; then
+ # Activate gcc9; enable errors on unset variables afterwards.
+ source /opt/rh/gcc-toolset-9/enable || exit 1
+ set -u
+ for cmd in "$@"; do
+ run_and_time "${cmd}"
+ done
+ echo "All specified dependencies installed!"
+ else
+ if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
+ echo "Installing build dependencies"
+ run_and_time install_build_prerequisites
+ else
+ echo "Skipping installation of build dependencies since
INSTALL_PREREQUISITES is not set"
+ fi
+ # Activate gcc9; enable errors on unset variables afterwards.
+ source /opt/rh/gcc-toolset-9/enable || exit 1
+ set -u
+ install_velox_deps
+ echo "All dependencies for Velox installed!"
+ dnf clean all
+ fi
+)
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
