This is an automated email from the ASF dual-hosted git repository.
yma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c4d9db271 [VL] Fix for centos9 build of Gluten (#6183)
c4d9db271 is described below
commit c4d9db2714442a905efe6e15f57c10ef4ca6e0f5
Author: deepa <[email protected]>
AuthorDate: Tue Jul 23 19:45:13 2024 +0530
[VL] Fix for centos9 build of Gluten (#6183)
* Merge conflicts resolution
* New build script updated
* Handled review comments
* removed BUILD_PROTOBUF from centos9
* Updated based on review comment
* Handling review comments
* Typo during merge added back
---------
Co-authored-by: Deepashree Gandhi <[email protected]>
---
.../backendsapi/velox/VeloxListenerApi.scala | 4 +-
.../gluten/utils/SharedLibraryLoaderCentos9.scala | 49 ++++++++++++++++++++++
dev/build_helper_functions.sh | 1 +
dev/package.sh | 10 ++++-
ep/build-velox/src/get_velox.sh | 40 ++++++++++++++++++
5 files changed, 102 insertions(+), 2 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index e5c3cb084..86925fd1d 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -66,6 +66,8 @@ class VeloxListenerApi extends ListenerApi {
new SharedLibraryLoaderUbuntu2004
} else if (systemName.contains("Ubuntu") &&
systemVersion.startsWith("22.04")) {
new SharedLibraryLoaderUbuntu2204
+ } else if (systemName.contains("CentOS") && systemVersion.startsWith("9"))
{
+ new SharedLibraryLoaderCentos9
} else if (systemName.contains("CentOS") && systemVersion.startsWith("8"))
{
new SharedLibraryLoaderCentos8
} else if (systemName.contains("CentOS") && systemVersion.startsWith("7"))
{
@@ -83,7 +85,7 @@ class VeloxListenerApi extends ListenerApi {
} else if (system.contains("tencentos") && system.contains("3.2")) {
new SharedLibraryLoaderCentos8
} else if (systemName.contains("Red Hat") &&
systemVersion.startsWith("9")) {
- new SharedLibraryLoaderCentos8
+ new SharedLibraryLoaderCentos9
} else if (systemName.contains("Red Hat") &&
systemVersion.startsWith("8")) {
new SharedLibraryLoaderCentos8
} else if (systemName.contains("Red Hat") &&
systemVersion.startsWith("7")) {
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
new file mode 100755
index 000000000..2d9ececb3
--- /dev/null
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.vectorized.JniLibLoader
+
+class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
+ override def loadLib(loader: JniLibLoader): Unit = {
+ loader
+ .newTransaction()
+ .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so",
false)
+ .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so",
false)
+ .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so",
false)
+ .loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
+ .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
+ .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
+ .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
+ .loadAndCreateLink("libboost_program_options.so.1.84.0",
"libboost_program_options.so", false)
+ .loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
+ .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so",
false)
+ .loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
+ .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
+ .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
+ .loadAndCreateLink("libglog.so.1", "libglog.so", false)
+ .loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false)
+ .loadAndCreateLink("libidn.so.12", "libidn.so", false)
+ .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
+ .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
+ .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+ .loadAndCreateLink("libre2.so.9", "libre2.so", false)
+ .loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
+ .commit()
+ }
+}
diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh
index 8559911c1..92ae5c29a 100644
--- a/dev/build_helper_functions.sh
+++ b/dev/build_helper_functions.sh
@@ -135,6 +135,7 @@ function setup_linux {
scripts/setup-ubuntu.sh
elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then
case "$LINUX_VERSION_ID" in
+ 9) scripts/setup-centos9.sh ;;
8) scripts/setup-centos8.sh ;;
7)
scripts/setup-centos7.sh
diff --git a/dev/package.sh b/dev/package.sh
index 7e7e793bd..db9125c2f 100755
--- a/dev/package.sh
+++ b/dev/package.sh
@@ -27,6 +27,12 @@ function process_setup_ubuntu_2204 {
cp
/usr/local/lib/{libhdfs3.so.1,libprotobuf.so.32,libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
}
+function process_setup_centos_9 {
+ cp
/lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libgsasl.so.7,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libidn.so.12,libntlm.so.0,libsodium.so.23}
$THIRDPARTY_LIB/
+ cp
/usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32}
$THIRDPARTY_LIB/
+ cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1} $THIRDPARTY_LIB/
+}
+
function process_setup_centos_8 {
cp
/usr/lib64/{libre2.so.0,libdouble-conversion.so.3,libevent-2.1.so.6,libdwarf.so.1,libgsasl.so.7,libicudata.so.60,libicui18n.so.60,libicuuc.so.60,libidn.so.11,libntlm.so.0,libsodium.so.23}
$THIRDPARTY_LIB/
cp
/usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32}
$THIRDPARTY_LIB/
@@ -56,7 +62,9 @@ if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then
process_setup_ubuntu_2204
fi
elif [ "$LINUX_OS" == "centos" ]; then
- if [ "$VERSION" == "8" ]; then
+ if [ "$VERSION" == "9" ]; then
+ process_setup_centos_9
+ elif [ "$VERSION" == "8" ]; then
process_setup_centos_8
elif [ "$VERSION" == "7" ]; then
process_setup_centos_7
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index d01d7eb12..818bdb99d 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -137,6 +137,45 @@ function process_setup_ubuntu {
sed -i '/run_and_time install_arrow/d' scripts/setup-ubuntu.sh
}
+function process_setup_centos9 {
+ # Allows other version of git already installed.
+ if [ -z "$(which git)" ]; then
+ dnf install -y -q --setopt=install_weak_deps=False git
+ fi
+ # make this function Reentrant
+ git checkout scripts/setup-centos9.sh
+ # No need to re-install git.
+
+ ensure_pattern_matched 'dnf_install' scripts/setup-centos9.sh
+ sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12
git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/'
scripts/setup-centos9.sh
+ sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}'
scripts/setup-centos9.sh
+ sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel
libuuid-devel' scripts/setup-centos9.sh
+
+ ensure_pattern_matched 'install_gflags' scripts/setup-centos9.sh
+ sed -i '/^function install_gflags.*/i function install_openssl {\n
wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make
install \n cd ..\n}\n' scripts/setup-centos9.sh
+
+ ensure_pattern_matched 'install_fbthrift' scripts/setup-centos9.sh
+ sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl'
scripts/setup-centos9.sh
+ sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure
CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh
+
+ if [ $ENABLE_HDFS == "ON" ]; then
+ sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd
"\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n
cmake_install\n}\n' scripts/setup-centos9.sh
+ sed -i '/^ run_and_time install_fbthrift/a \ run_and_time
install_libhdfs3' scripts/setup-centos9.sh
+ sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\'
scripts/setup-centos9.sh
+ fi
+ sed -i "s/yum -y install/sudo yum -y install/"
${VELOX_HOME}/scripts/setup-adapters.sh
+ if [ $ENABLE_S3 == "ON" ]; then
+ sed -i '/^ run_and_time install_fbthrift/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos9.sh
+ fi
+ if [ $ENABLE_GCS == "ON" ]; then
+ sed -i '/^ run_and_time install_fbthrift/a \ \
'${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos9.sh
+ fi
+ if [ $ENABLE_ABFS == "ON" ]; then
+ sed -i '/^ run_and_time install_fbthrift/a \ \ export
AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh
abfs' scripts/setup-centos9.sh
+ fi
+}
+
+
function process_setup_centos8 {
# Allows other version of git already installed.
if [ -z "$(which git)" ]; then
@@ -287,6 +326,7 @@ function setup_linux {
process_setup_ubuntu
elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then
case "$LINUX_VERSION_ID" in
+ 9) process_setup_centos9 ;;
8) process_setup_centos8 ;;
7) process_setup_centos7 ;;
*)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
