This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 58db130446 [VL] Clean up some legacy code and correct minimum required
GCC version (#7865)
58db130446 is described below
commit 58db1304468e475425c126685424cbe7982600ce
Author: PHILO-HE <[email protected]>
AuthorDate: Fri Nov 8 17:08:14 2024 +0800
[VL] Clean up some legacy code and correct minimum required GCC version
(#7865)
---
.github/workflows/build_bundle_package.yml | 1 -
.github/workflows/velox_backend.yml | 95 ------------------------------
.github/workflows/velox_backend_cache.yml | 7 ---
dev/ci-velox-buildshared-centos-8.sh | 2 -
dev/ci-velox-buildstatic-centos-7.sh | 2 -
dev/vcpkg/README.md | 2 +-
docs/developers/NewToGluten.md | 2 +-
ep/build-velox/src/get_velox.sh | 11 ----
8 files changed, 2 insertions(+), 120 deletions(-)
diff --git a/.github/workflows/build_bundle_package.yml
b/.github/workflows/build_bundle_package.yml
index d4f7046f2b..a287206566 100644
--- a/.github/workflows/build_bundle_package.yml
+++ b/.github/workflows/build_bundle_package.yml
@@ -51,7 +51,6 @@ jobs:
- name: Build Gluten velox third party
run: |
df -a
- yum install ccache -y
cd $GITHUB_WORKSPACE/
bash dev/ci-velox-buildstatic-centos-7.sh
- name: Upload native libs
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 737895be57..c316dba6f8 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -69,7 +69,6 @@ jobs:
- name: Build Gluten native libraries
run: |
df -a
- yum install ccache -y
cd $GITHUB_WORKSPACE/
bash dev/ci-velox-buildstatic-centos-7.sh
- name: "Save ccache"
@@ -544,14 +543,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.2.2 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.2
@@ -590,14 +581,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.2.2 (slow tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.2
@@ -629,14 +612,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.3.1 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.3
@@ -676,14 +651,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.3.1 (slow tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.3
@@ -716,14 +683,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.4.3 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.4
@@ -763,14 +722,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.4.3 (slow tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.4
@@ -803,14 +754,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.5.3 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
@@ -849,14 +792,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.5.3 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5-scala2.13
@@ -895,14 +830,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.5.3 (slow tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
@@ -935,14 +862,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.5.3 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
@@ -980,14 +899,6 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Update mirror list
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- - name: Setup build dependency
- run: |
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Prepare spark.test.home for Spark 3.5.3 (slow tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
@@ -1015,12 +926,6 @@ jobs:
key: ccache-centos8-release-default-${{github.sha}}
restore-keys: |
ccache-centos8-release-default
- - name: Setup java and maven
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- $SETUP install_maven
- name: Build Gluten native libraries
run: |
df -a
diff --git a/.github/workflows/velox_backend_cache.yml
b/.github/workflows/velox_backend_cache.yml
index 71a7a5840b..e969ffb741 100644
--- a/.github/workflows/velox_backend_cache.yml
+++ b/.github/workflows/velox_backend_cache.yml
@@ -44,7 +44,6 @@ jobs:
- name: Build Gluten native libraries
run: |
df -a
- yum install ccache -y
bash dev/ci-velox-buildstatic-centos-7.sh
- name: Save Ccache
uses: actions/cache/save@v3
@@ -65,12 +64,6 @@ jobs:
key: ccache-centos8-release-default-${{github.sha}}
restore-keys: |
ccache-centos8-release-default
- - name: Setup java and maven
- run: |
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
- yum install sudo patch java-1.8.0-openjdk-devel wget -y
- bash .github/workflows/util/setup_helper.sh install_maven
- name: Build Gluten native libraries
run: |
df -a
diff --git a/dev/ci-velox-buildshared-centos-8.sh
b/dev/ci-velox-buildshared-centos-8.sh
index f295632ffe..ff9a62b798 100755
--- a/dev/ci-velox-buildshared-centos-8.sh
+++ b/dev/ci-velox-buildshared-centos-8.sh
@@ -2,8 +2,6 @@
set -e
-# TODO: will be removed after docker is updated.
-dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
source /opt/rh/gcc-toolset-11/enable
./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF
--enable_ep_cache=OFF --build_tests=ON \
--build_examples=ON --build_benchmarks=ON
diff --git a/dev/ci-velox-buildstatic-centos-7.sh
b/dev/ci-velox-buildstatic-centos-7.sh
index 76bd33d7ff..6895891a98 100755
--- a/dev/ci-velox-buildstatic-centos-7.sh
+++ b/dev/ci-velox-buildstatic-centos-7.sh
@@ -2,8 +2,6 @@
set -e
-# TODO: will be removed after docker is updated.
-sudo yum install -y devtoolset-11
source /opt/rh/devtoolset-11/enable
export NUM_THREADS=4
./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF
--build_tests=OFF --build_benchmarks=OFF \
diff --git a/dev/vcpkg/README.md b/dev/vcpkg/README.md
index 229e11cc72..986a8ec689 100644
--- a/dev/vcpkg/README.md
+++ b/dev/vcpkg/README.md
@@ -15,7 +15,7 @@ Please install build depends on your system to compile all
libraries:
sudo $GLUTEN_REPO/dev/vcpkg/setup-build-depends.sh
```
-For CentOS user, gcc 11 needs to be enabled manually before next step:
+GCC-11 is the minimum required compiler. It needs to be enabled beforehand.
Take Centos-7/8 as example:
``` sh
# CentOS 8
diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md
index 8228c919e8..347213a418 100644
--- a/docs/developers/NewToGluten.md
+++ b/docs/developers/NewToGluten.md
@@ -58,7 +58,7 @@
spark.executor.extraJavaOptions=-Dio.netty.tryReflectionSetAccessible=true
[Maven Dowload Page](https://maven.apache.org/docs/history.html)
And then set the environment setting.
-## GCC 9.4 or above
+## GCC 11 or above
# Compile gluten using debug mode
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index a3220cbb34..37dc20aedb 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -69,17 +69,6 @@ function process_setup_ubuntu {
# Do not install libunwind which can cause interruption when catching native
exception.
ensure_pattern_matched '\${SUDO} apt install -y libunwind-dev'
scripts/setup-ubuntu.sh
sed -i 's/${SUDO} apt install -y libunwind-dev//' scripts/setup-ubuntu.sh
- # Overwrite gcc installed by build-essential.
- ensure_pattern_matched '\${SUDO} pip3 install cmake==3.28.3'
scripts/setup-ubuntu.sh
- sed -i '/^ ${SUDO} pip3 install cmake==3.28.3/a\
- \VERSION=`cat /etc/os-release | grep VERSION_ID`\
- if [[ $VERSION =~ "20.04" ]]; then\
- sudo apt install -y software-properties-common\
- sudo add-apt-repository ppa:ubuntu-toolchain-r/test\
- sudo apt update && sudo apt install -y gcc-11 g++-11\
- sudo ln -sf /usr/bin/gcc-11 /usr/bin/gcc\
- sudo ln -sf /usr/bin/g++-11 /usr/bin/g++\
- fi' scripts/setup-ubuntu.sh
ensure_pattern_matched 'ccache' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libiberty-dev \\' scripts/setup-ubuntu.sh
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
