This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8e8b6b383f [GLUTEN-12412][VL] Migrate x86/arm vcpkg native lib build
from CentOS 7 to AlmaLinux 8 (#12663)
8e8b6b383f is described below
commit 8e8b6b383fc934e1e64a1c93a3ebda30b5930240
Author: Kushaagra Shrivastava <[email protected]>
AuthorDate: Tue Sep 22 00:14:08 2026 +0530
[GLUTEN-12412][VL] Migrate x86/arm vcpkg native lib build from CentOS 7 to
AlmaLinux 8 (#12663)
---
.github/workflows/build_bundle_package.yml | 25 ++--
.github/workflows/build_release.yml | 5 +-
.github/workflows/velox_backend_ansi.yml | 14 +-
.github/workflows/velox_backend_cache.yml | 30 +++--
.github/workflows/velox_backend_enhanced.yml | 32 ++---
.github/workflows/velox_backend_x86.yml | 143 +++++++--------------
.github/workflows/velox_nightly.yml | 38 +++---
...velox-buildstatic-centos-8-enhanced-features.sh | 2 +
dev/ci-velox-buildstatic-centos-8.sh | 2 +
.../Dockerfile.almalinux8-gcc13-static-build | 2 +-
dev/vcpkg/README.md | 7 +-
11 files changed, 131 insertions(+), 169 deletions(-)
diff --git a/.github/workflows/build_bundle_package.yml
b/.github/workflows/build_bundle_package.yml
index 1a5f9a832e..1e592f054d 100644
--- a/.github/workflows/build_bundle_package.yml
+++ b/.github/workflows/build_bundle_package.yml
@@ -42,6 +42,7 @@ on:
jobs:
build-native-lib:
runs-on: ubuntu-22.04
+ container: apache/gluten:vcpkg-almalinux-8-gcc13
steps:
- uses: actions/checkout@v7
with:
@@ -50,20 +51,18 @@ jobs:
uses: actions/cache/restore@v3
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{github.sha}}
+ key: ccache-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-centos7-release-default
+ ccache-almalinux8-release-default
- name: Build Gluten velox third party
run: |
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
- df -a
- cd /work
- export CCACHE_DIR=/work/.ccache
- bash dev/ci-velox-buildstatic-centos-7.sh
- ccache -s
- mkdir -p /work/.m2/repository/org/apache/arrow/
- cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
- "
+ yum install bison -y
+ df -a
+ export CCACHE_DIR=${{ env.CCACHE_DIR }}
+ bash dev/ci-velox-buildstatic-centos-8.sh
+ ccache -s
+ mkdir -p ${{ github.workspace }}/.m2/repository/org/apache/arrow/
+ cp -r /root/.m2/repository/org/apache/arrow/* ${{ github.workspace
}}/.m2/repository/org/apache/arrow/
- name: Upload native libs
uses: actions/upload-artifact@v7
with:
@@ -74,7 +73,7 @@ jobs:
uses: actions/upload-artifact@v7
with:
path: ${{ github.workspace }}/.m2/repository/org/apache/arrow/
- name: velox-arrow-jar-centos-7-${{github.sha}}
+ name: velox-arrow-jar-centos-8-${{github.sha}}
build-bundle-package-centos8:
needs: build-native-lib
@@ -92,7 +91,7 @@ jobs:
- name: Download All Arrow Jar Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-arrow-jar-centos-7-${{github.sha}}
+ name: velox-arrow-jar-centos-8-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
diff --git a/.github/workflows/build_release.yml
b/.github/workflows/build_release.yml
index 88d3c429d4..f680132fb1 100644
--- a/.github/workflows/build_release.yml
+++ b/.github/workflows/build_release.yml
@@ -28,9 +28,10 @@ jobs:
- uses: actions/checkout@v7
- name: Build
run: |
- docker pull apache/gluten:vcpkg-centos-7-gcc13
- docker run -v $GITHUB_WORKSPACE:/workspace -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
+ docker pull apache/gluten:vcpkg-almalinux-8-gcc13
+ docker run -v $GITHUB_WORKSPACE:/workspace -w /workspace
apache/gluten:vcpkg-almalinux-8-gcc13 bash -c "
set -e
+ yum install bison -y
cd /workspace
bash dev/release/build-release.sh
"
diff --git a/.github/workflows/velox_backend_ansi.yml
b/.github/workflows/velox_backend_ansi.yml
index 64c158f96d..99b0df36b0 100644
--- a/.github/workflows/velox_backend_ansi.yml
+++ b/.github/workflows/velox_backend_ansi.yml
@@ -120,21 +120,21 @@ jobs:
uses: actions/cache/restore@v4
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{github.sha}}
+ key: ccache-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-centos7-release-default
+ ccache-almalinux8-release-default
- name: Build Gluten native libraries
run: |
- docker pull apache/gluten:vcpkg-centos-7-gcc13
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
+ docker pull apache/gluten:vcpkg-almalinux-8-gcc13
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-almalinux-8-gcc13 bash -c "
set -e
- yum install tzdata -y
+ yum install tzdata bison -y
cd /work
export CCACHE_DIR=/work/.ccache
export CCACHE_MAXSIZE=1G
mkdir -p /work/.ccache
ccache -sz
- bash dev/ci-velox-buildstatic-centos-7.sh
+ bash dev/ci-velox-buildstatic-centos-8.sh
ccache -s
"
- name: Save ccache
@@ -142,7 +142,7 @@ jobs:
uses: actions/cache/save@v4
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{github.sha}}
+ key: ccache-almalinux8-release-default-${{github.sha}}
- uses: actions/upload-artifact@v7
with:
name: velox-native-lib-ansi-${{github.sha}}
diff --git a/.github/workflows/velox_backend_cache.yml
b/.github/workflows/velox_backend_cache.yml
index 054fb98768..632cc8ab78 100644
--- a/.github/workflows/velox_backend_cache.yml
+++ b/.github/workflows/velox_backend_cache.yml
@@ -29,7 +29,7 @@ concurrency:
cancel-in-progress: false
jobs:
- cache-native-lib-centos-7:
+ cache-native-lib-centos-8-static:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
steps:
@@ -38,28 +38,29 @@ jobs:
uses: actions/cache/restore@v3
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{github.sha}}
+ key: ccache-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-centos7-release-default
+ ccache-almalinux8-release-default
- name: Build Gluten native libraries
run: |
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-almalinux-8-gcc13 bash -c "
+ yum install bison -y
export CCACHE_DIR=/work/.ccache
export CCACHE_MAXSIZE=1G
mkdir -p /work/.ccache
- bash dev/ci-velox-buildstatic-centos-7.sh
+ bash dev/ci-velox-buildstatic-centos-8.sh
"
- name: Save Ccache
uses: actions/cache/save@v3
id: ccache
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{github.sha}}
+ key: ccache-almalinux8-release-default-${{github.sha}}
- name: Save Ccache to Apache Stash
uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{
hashFiles('ep/build-velox/src/**') }}
+ key: ccache-almalinux8-release-default-${{
hashFiles('ep/build-velox/src/**') }}
include-hidden-files: true
cache-native-lib-centos-8:
@@ -130,7 +131,7 @@ jobs:
key: ccache-centos9-release-shared-${{runner.arch}}-${{
hashFiles('ep/build-velox/src/**') }}
include-hidden-files: true
- cache-enhanced-native-lib-centos-7:
+ cache-enhanced-native-lib-centos-8-static:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
steps:
@@ -139,28 +140,29 @@ jobs:
uses: actions/cache/restore@v3
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-enhanced-centos7-release-default-${{github.sha}}
+ key: ccache-enhanced-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-enhanced-centos7-release-default
+ ccache-enhanced-almalinux8-release-default
- name: Build Gluten native libraries
run: |
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-almalinux-8-gcc13 bash -c "
+ yum install bison -y
export CCACHE_DIR=/work/.ccache
export CCACHE_MAXSIZE=1G
mkdir -p /work/.ccache
- bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
+ bash dev/ci-velox-buildstatic-centos-8-enhanced-features.sh
"
- name: Save Ccache
uses: actions/cache/save@v3
id: ccache
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-enhanced-centos7-release-default-${{github.sha}}
+ key: ccache-enhanced-almalinux8-release-default-${{github.sha}}
- name: Save Ccache to Apache Stash
uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-enhanced-centos7-release-default-${{
hashFiles('ep/build-velox/src/**') }}
+ key: ccache-enhanced-almalinux8-release-default-${{
hashFiles('ep/build-velox/src/**') }}
include-hidden-files: true
cache-shared-lib-centos-9-cudf:
diff --git a/.github/workflows/velox_backend_enhanced.yml
b/.github/workflows/velox_backend_enhanced.yml
index fe51351786..a8c9976e78 100644
--- a/.github/workflows/velox_backend_enhanced.yml
+++ b/.github/workflows/velox_backend_enhanced.yml
@@ -79,7 +79,7 @@ jobs:
echo "shims35=$(match
'^(shims/(spark35|common)/|gluten-ut/spark35/)')" >>
$GITHUB_OUTPUT
echo "shims40=$(match
'^(shims/(spark40|common)/|gluten-ut/spark40/)')" >>
$GITHUB_OUTPUT
- build-native-lib-centos-7:
+ build-native-lib-centos-8:
needs: detect-changes
runs-on: ubuntu-22.04
steps:
@@ -89,28 +89,28 @@ jobs:
uses:
apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-enhanced-centos7-release-default-${{
hashFiles('ep/build-velox/src/**') }}
+ key: ccache-enhanced-almalinux8-release-default-${{
hashFiles('ep/build-velox/src/**') }}
- name: Get Ccache from actions/cache (fallback on Stash miss)
if: steps.ccache-stash.outputs.stash-hit != 'true'
uses: actions/cache/restore@v4
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-enhanced-centos7-release-default-${{github.sha}}
+ key: ccache-enhanced-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-enhanced-centos7-release-default
+ ccache-enhanced-almalinux8-release-default
- name: Build Gluten native libraries
run: |
- docker pull apache/gluten:vcpkg-centos-7-gcc13
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
+ docker pull apache/gluten:vcpkg-almalinux-8-gcc13
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-almalinux-8-gcc13 bash -c "
set -e
- yum install tzdata -y
+ yum install tzdata bison -y
df -a
cd /work
export CCACHE_DIR=/work/.ccache
export CCACHE_MAXSIZE=1G
mkdir -p /work/.ccache
ccache -sz
- bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
+ bash dev/ci-velox-buildstatic-centos-8-enhanced-features.sh
ccache -s
"
@@ -119,16 +119,16 @@ jobs:
uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-enhanced-centos7-release-default-${{
hashFiles('ep/build-velox/src/**') }}
+ key: ccache-enhanced-almalinux8-release-default-${{
hashFiles('ep/build-velox/src/**') }}
include-hidden-files: true
- uses: actions/upload-artifact@v7
with:
- name: velox-native-lib-enhanced-centos-7-${{github.sha}}
+ name: velox-native-lib-enhanced-centos-8-${{github.sha}}
path: ./cpp/build/releases/
if-no-files-found: error
spark-test-spark35:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims35 == 'true' ||
@@ -140,7 +140,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-enhanced-centos-7-${{github.sha}}
+ name: velox-native-lib-enhanced-centos-8-${{github.sha}}
path: ./cpp/build/releases
- name: Prepare spark.test.home for Spark 3.5.5 (other tests)
run: |
@@ -185,7 +185,7 @@ jobs:
path: /tmp/tpch-approved-plan/**
spark-test-spark35-slow:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims35 == 'true' ||
@@ -197,7 +197,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-enhanced-centos-7-${{github.sha}}
+ name: velox-native-lib-enhanced-centos-8-${{github.sha}}
path: ./cpp/build/releases
- name: Build and Run unit test for Spark 3.5.5 (slow tests)
run: |
@@ -229,7 +229,7 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark40:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims40 == 'true' ||
@@ -247,7 +247,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-enhanced-centos-7-${{github.sha}}
+ name: velox-native-lib-enhanced-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Prepare
run: |
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 666b9f72b1..ecc4d8b73e 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -144,56 +144,56 @@ jobs:
echo "shims41=$(match
'^(shims/(spark41|common)/|gluten-ut/spark41/)')" >>
$GITHUB_OUTPUT
echo "tools_it=$(match '^tools/gluten-it/')"
>> $GITHUB_OUTPUT
- build-native-lib-centos-7:
+ build-native-lib-centos-8:
needs: detect-changes
runs-on: ubuntu-22.04
+ container: apache/gluten:vcpkg-almalinux-8-gcc13
steps:
+ - name: Install CI tools (temporary - until image is rebuilt with
gh/jq/bison/python3.8)
+ run: |
+ dnf install -y jq bison tzdata
+ dnf module -y install python38 && alternatives --set python3
/usr/bin/python3.8
+ dnf config-manager --add-repo
https://cli.github.com/packages/rpm/gh-cli.repo
+ dnf install -y gh --setopt=gpgcheck=0
- uses: actions/checkout@v7
- name: Get Ccache from Apache Stash
id: ccache-stash
uses:
apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{
hashFiles('ep/build-velox/src/**') }}
+ key: ccache-almalinux8-release-default-${{
hashFiles('ep/build-velox/src/**') }}
- name: Get Ccache from actions/cache (fallback on Stash miss)
if: steps.ccache-stash.outputs.stash-hit != 'true'
uses: actions/cache/restore@v4
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{github.sha}}
+ key: ccache-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-centos7-release-default
+ ccache-almalinux8-release-default
- name: Build Gluten native libraries
run: |
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
- set -e
- yum install tzdata -y
- df -a
- cd /work
- export CCACHE_DIR=/work/.ccache
- export CCACHE_MAXSIZE=1G
- mkdir -p /work/.ccache
- ccache -sz
- bash dev/ci-velox-buildstatic-centos-7.sh
- ccache -s
- "
+ df -a
+ export CCACHE_MAXSIZE=1G
+ ccache -sz
+ bash dev/ci-velox-buildstatic-centos-8.sh
+ ccache -s
- name: "Save ccache"
if: always()
uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{
hashFiles('ep/build-velox/src/**') }}
+ key: ccache-almalinux8-release-default-${{
hashFiles('ep/build-velox/src/**') }}
include-hidden-files: true
- uses: actions/upload-artifact@v7
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
if-no-files-found: error
retention-days: 1
tpc-test-ubuntu:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.cpp == 'true' ||
needs.detect-changes.outputs.java == 'true' ||
@@ -245,7 +245,7 @@ jobs:
- name: Download All Native Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Setup tzdata
run: |
@@ -329,7 +329,7 @@ jobs:
fi
tpc-test-centos8:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.cpp == 'true' ||
needs.detect-changes.outputs.java == 'true' ||
@@ -375,7 +375,7 @@ jobs:
- name: Download All Native Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Update mirror list
run: |
@@ -424,55 +424,8 @@ jobs:
GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \
--local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
- tpc-test-centos7:
- needs: [detect-changes, build-native-lib-centos-7]
- if: >-
- needs.detect-changes.outputs.cpp == 'true' ||
- needs.detect-changes.outputs.java == 'true' ||
- needs.detect-changes.outputs.tools_it == 'true'
- strategy:
- fail-fast: false
- matrix:
- spark: [ "spark-3.5" ]
- java: [ "java-8" ]
- runs-on: ubuntu-22.04
- steps:
- - uses: actions/checkout@v7
- - name: Download All Native Artifacts
- uses: actions/download-artifact@v8
- with:
- name: velox-native-lib-centos-7-${{github.sha}}
- path: ./cpp/build/releases/
- - name: Build and run TPCH/DS tests
- run: |
- docker pull centos:7
- docker run -v $GITHUB_WORKSPACE:/work -v
/$GITHUB_WORKSPACE/.m2:/root/.m2/ -w /work \
- -e matrix.java=${{ matrix.java }} -e matrix.spark=${{ matrix.spark
}} \
- centos:7 \
- bash -c "
- set -e
- sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-*
|| true
- sed -i -e
's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* || true
-
- yum update -y && yum install -y java-1.8.0-openjdk-devel wget
tzdata python3-pip
- # Set environment variables
- export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
-
- # Build gluten
- build/mvn -ntp clean install -P${{ matrix.spark }} -P${{
matrix.java }} -Pbackends-velox -DskipTests
- # Build gluten-it
- cd /work/tools/gluten-it
- /work/build/mvn -ntp clean install -P${{ matrix.spark }} -P${{
matrix.java }}
-
- # Run TPC-H / TPC-DS
- GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
- GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
- "
-
tpc-test-ubuntu-oom:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
# Chaos/stress tier: this job probes native memory-pressure behaviour, so
# per-PR it only runs when native code changed (cpp=true). For java-only
# changes its ~60 runner-minutes rarely produce PR-specific signal; the
@@ -502,7 +455,7 @@ jobs:
- name: Download All Native Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Setup java
run: |
@@ -589,7 +542,7 @@ jobs:
-d=IO_THREADS:0,spark.gluten.sql.columnar.backend.velox.IOThreads=0
tpc-test-ubuntu-randomkill:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
# Chaos/stress tier (see tpc-test-ubuntu-oom): random task kills mainly
# probe native/JNI shutdown robustness, and at 3 shards this is the single
# most expensive TPC job (~190 runner-minutes). Per-PR only for native
@@ -620,7 +573,7 @@ jobs:
- name: Download All Native Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Setup java
run: |
@@ -644,7 +597,7 @@ jobs:
--data-gen=skip --random-kill-tasks --no-session-reuse
--shard=${{ matrix.shard }}/3
tpc-test-centos8-uniffle:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.cpp == 'true' ||
needs.detect-changes.outputs.java == 'true' ||
@@ -662,7 +615,7 @@ jobs:
- name: Download All Native Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Install Uniffle ${{ matrix.uniffle }}
run: |
@@ -694,7 +647,7 @@ jobs:
--local --preset=velox-with-uniffle --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
tpc-test-ubuntu-2204-celeborn:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.cpp == 'true' ||
needs.detect-changes.outputs.java == 'true' ||
@@ -719,7 +672,7 @@ jobs:
- name: Download All Native Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Update mirror list
run: |
@@ -778,7 +731,7 @@ jobs:
fi
spark-test-spark34:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims34 == 'true' ||
@@ -794,7 +747,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Prepare spark.test.home for Spark 3.4.4 (other tests)
run: |
@@ -849,7 +802,7 @@ jobs:
path: /tmp/tpch-approved-plan/**
spark-test-spark34-slow:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims34 == 'true' ||
@@ -865,7 +818,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Build and Run unit test for Spark 3.4.4 (slow tests, ${{
matrix.suite }})
run: |
@@ -901,7 +854,7 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark35:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims35 == 'true' ||
@@ -919,7 +872,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Prepare
run: |
@@ -973,7 +926,7 @@ jobs:
path: /tmp/tpch-approved-plan/**
spark-test-spark35-scala213:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims35 == 'true' ||
@@ -991,7 +944,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Prepare
run: |
@@ -1039,7 +992,7 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark35-slow:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims35 == 'true' ||
@@ -1057,7 +1010,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Build and Run unit test for Spark 3.5.5 (slow tests, ${{
matrix.suite }})
run: |
@@ -1292,7 +1245,7 @@ jobs:
$MVN_CMD clean test-compile -Pspark-4.0 -Pscala-2.13
-Pbackends-velox -Pspark-ut -Piceberg,iceberg-test,delta,paimon -Pfast-build
spark-test-spark40:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims40 == 'true' ||
@@ -1310,7 +1263,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Prepare
run: |
@@ -1358,7 +1311,7 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark40-slow:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims40 == 'true' ||
@@ -1376,7 +1329,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Build and Run unit test for Spark 4.0 (slow tests, ${{
matrix.suite }})
run: |
@@ -1411,7 +1364,7 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark41:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims41 == 'true' ||
@@ -1433,7 +1386,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Prepare
run: |
@@ -1486,7 +1439,7 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark41-slow:
- needs: [detect-changes, build-native-lib-centos-7]
+ needs: [detect-changes, build-native-lib-centos-8]
if: >-
needs.detect-changes.outputs.java == 'true' ||
needs.detect-changes.outputs.shims41 == 'true' ||
@@ -1505,7 +1458,7 @@ jobs:
- name: Download All Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-native-lib-centos-7-${{github.sha}}
+ name: velox-native-lib-centos-8-${{github.sha}}
path: ./cpp/build/releases/
- name: Build and Run unit test for Spark 4.1 (slow tests, ${{
matrix.suite }})
run: |
diff --git a/.github/workflows/velox_nightly.yml
b/.github/workflows/velox_nightly.yml
index f55d5cd187..91f74a6491 100644
--- a/.github/workflows/velox_nightly.yml
+++ b/.github/workflows/velox_nightly.yml
@@ -43,17 +43,18 @@ jobs:
uses: actions/cache/restore@v4
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos7-release-default-${{github.sha}}
+ key: ccache-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-centos7-release-default
+ ccache-almalinux8-release-default
- name: Build Gluten velox third party
run: |
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-almalinux-8-gcc13 bash -c "
+ yum install bison -y
df -a
cd /work
export CCACHE_DIR=/work/.ccache
export CCACHE_MAXSIZE=1G
- bash dev/ci-velox-buildstatic-centos-7.sh
+ bash dev/ci-velox-buildstatic-centos-8.sh
ccache -s
mkdir -p /work/.m2/repository/org/apache/arrow/
cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
@@ -68,7 +69,7 @@ jobs:
uses: actions/upload-artifact@v7
with:
path: .m2/repository/org/apache/arrow/
- name: velox-arrow-jar-centos-7-${{github.sha}}
+ name: velox-arrow-jar-centos-8-x86-${{github.sha}}
build-bundle-package-centos8-jdk8-x86:
if: ${{ startsWith(github.repository, 'apache/') }}
@@ -85,7 +86,7 @@ jobs:
- name: Download All Arrow Jar Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-arrow-jar-centos-7-${{github.sha}}
+ name: velox-arrow-jar-centos-8-x86-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
@@ -122,7 +123,7 @@ jobs:
- name: Download All Arrow Jar Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-arrow-jar-centos-7-${{github.sha}}
+ name: velox-arrow-jar-centos-8-x86-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
@@ -161,7 +162,7 @@ jobs:
- name: Download All Arrow Jar Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-arrow-jar-centos-7-${{github.sha}}
+ name: velox-arrow-jar-centos-8-x86-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- name: Setup java
run: |
@@ -282,16 +283,16 @@ jobs:
build-native-lib-centos-8-arm64:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-24.04-arm
- container: apache/gluten:vcpkg-centos-8-gcc13
+ container: apache/gluten:vcpkg-almalinux-8-gcc13
steps:
- uses: actions/checkout@v7
- name: Get Ccache
uses: actions/cache/restore@v4
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos8-release-default-${{runner.arch}}-${{github.sha}}
+ key:
ccache-almalinux8-release-default-${{runner.arch}}-${{github.sha}}
restore-keys: |
- ccache-centos8-release-default-${{runner.arch}}
+ ccache-almalinux8-release-default-${{runner.arch}}
- name: Build Gluten native libraries
run: |
df -a
@@ -306,7 +307,7 @@ jobs:
id: ccache
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos8-release-default-${{runner.arch}}-${{github.sha}}
+ key:
ccache-almalinux8-release-default-${{runner.arch}}-${{github.sha}}
- uses: actions/upload-artifact@v7
with:
name: velox-native-lib-centos-8-${{github.sha}}
@@ -528,16 +529,17 @@ jobs:
uses: actions/cache/restore@v4
with:
path: '${{ env.CCACHE_DIR }}'
- key: ccache-enhanced-centos7-release-default-${{github.sha}}
+ key: ccache-enhanced-almalinux8-release-default-${{github.sha}}
restore-keys: |
- ccache-enhanced-centos7-release-default
+ ccache-enhanced-almalinux8-release-default
- name: Build Gluten velox third party
run: |
- docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7-gcc13 bash -c "
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-almalinux-8-gcc13 bash -c "
+ yum install bison -y
df -a
cd /work
export CCACHE_DIR=/work/.ccache
- bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
+ bash dev/ci-velox-buildstatic-centos-8-enhanced-features.sh
ccache -s
mkdir -p /work/.m2/repository/org/apache/arrow/
cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
@@ -552,7 +554,7 @@ jobs:
uses: actions/upload-artifact@v7
with:
path: .m2/repository/org/apache/arrow/
- name: velox-arrow-jar-centos-7-enhanced-${{github.sha}}
+ name: velox-arrow-jar-centos-8-enhanced-${{github.sha}}
build-bundle-enhanced-package-centos8-jdk17:
if: ${{ startsWith(github.repository, 'apache/') }}
@@ -569,7 +571,7 @@ jobs:
- name: Download All Arrow Jar Artifacts
uses: actions/download-artifact@v8
with:
- name: velox-arrow-jar-centos-7-enhanced-${{github.sha}}
+ name: velox-arrow-jar-centos-8-enhanced-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- name: Setup java and maven
run: |
diff --git a/dev/ci-velox-buildstatic-centos-8-enhanced-features.sh
b/dev/ci-velox-buildstatic-centos-8-enhanced-features.sh
index c995e75e5d..176a13bd18 100755
--- a/dev/ci-velox-buildstatic-centos-8-enhanced-features.sh
+++ b/dev/ci-velox-buildstatic-centos-8-enhanced-features.sh
@@ -17,6 +17,8 @@
set -e
+source /opt/rh/gcc-toolset-13/enable
+
if [ "$(uname -m)" = "aarch64" ]; then
export CPU_TARGET="aarch64";
export VCPKG_FORCE_SYSTEM_BINARIES=1;
diff --git a/dev/ci-velox-buildstatic-centos-8.sh
b/dev/ci-velox-buildstatic-centos-8.sh
index b23225d397..f5cd4ed884 100644
--- a/dev/ci-velox-buildstatic-centos-8.sh
+++ b/dev/ci-velox-buildstatic-centos-8.sh
@@ -17,6 +17,8 @@
set -e
+source /opt/rh/gcc-toolset-13/enable
+
if [ "$(uname -m)" = "aarch64" ]; then
export CPU_TARGET="aarch64";
export VCPKG_FORCE_SYSTEM_BINARIES=1;
diff --git a/dev/docker/Dockerfile.almalinux8-gcc13-static-build
b/dev/docker/Dockerfile.almalinux8-gcc13-static-build
index 954d9403d9..3c229157f9 100644
--- a/dev/docker/Dockerfile.almalinux8-gcc13-static-build
+++ b/dev/docker/Dockerfile.almalinux8-gcc13-static-build
@@ -44,7 +44,7 @@ RUN set -ex; \
dnf install -y ccache; \
# gcc-toolset-13 ships GCC 13 on RHEL8-family via Red Hat Software
Collections. \
dnf install -y --setopt=install_weak_deps=False gcc-toolset-13; \
- dnf install -y java-1.8.0-openjdk-devel patch git wget perl python3
automake libtool flex; \
+ dnf install -y java-1.8.0-openjdk-devel patch git wget perl python3
automake libtool flex bison; \
dnf -y --enablerepo=powertools install autoconf-archive ninja-build; \
# Install a recent CMake binary — the version in EPEL 8 is too old for
Velox. \
wget
https://github.com/Kitware/CMake/releases/download/v3.31.12/cmake-3.31.12-linux-$(uname
-m).sh; \
diff --git a/dev/vcpkg/README.md b/dev/vcpkg/README.md
index 549d70295e..f7edf49090 100644
--- a/dev/vcpkg/README.md
+++ b/dev/vcpkg/README.md
@@ -9,9 +9,10 @@ except for those overridden in `vcpkg.json`,
`vcpkg-configuration.json`, and ove
## Build in docker
For main branch code, you can follow the commands below.
-- Pull the docker image: `docker pull apache/gluten:vcpkg-centos-7-gcc13`
-- Build native code: `bash dev/ci-velox-buildstatic-centos-7.sh`
-- Build JVM code: `mvn clean install -Pbackends-velox -Pspark-3.5 -DskipTests`
+
+- Pull the docker image: `docker pull apache/gluten:vcpkg-almalinux-8-gcc13`
+- Build native code: `bash dev/ci-velox-buildstatic-centos-8.sh`
+- Build JVM code: `./build/mvn clean install -Pbackends-velox -Pspark-3.5
-DskipTests`
The gluten packages will be placed in
`$GLUTEN_REPO/package/target/gluten-velox-bundle-*.jar`.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
