This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2649fa7b34 [GLUTEN-8046][VL] CI: fix velox cache/bundle package script
(#8051)
2649fa7b34 is described below
commit 2649fa7b34edf99268bff0d95d9d9621683f839c
Author: Yuan <[email protected]>
AuthorDate: Wed Nov 27 09:06:50 2024 +0800
[GLUTEN-8046][VL] CI: fix velox cache/bundle package script (#8051)
This patch fixes the cache/build script due to nodejs version update in GHA
This patch also adds centos7 tests back
fixes #8046
Signed-off-by: Yuan Zhou <[email protected]>
---
.github/workflows/build_bundle_package.yml | 19 +++++---
.github/workflows/velox_backend.yml | 78 +++++++++++++++++++++++++++---
.github/workflows/velox_backend_cache.yml | 8 +--
3 files changed, 86 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/build_bundle_package.yml
b/.github/workflows/build_bundle_package.yml
index 9b94d302f9..c7ddd8748b 100644
--- a/.github/workflows/build_bundle_package.yml
+++ b/.github/workflows/build_bundle_package.yml
@@ -38,9 +38,8 @@ on:
jobs:
build-native-lib:
runs-on: ubuntu-20.04
- container: apache/gluten:vcpkg-centos-7
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v4
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -50,9 +49,15 @@ jobs:
ccache-centos7-release-default
- name: Build Gluten velox third party
run: |
- df -a
- cd $GITHUB_WORKSPACE/
- bash dev/ci-velox-buildstatic-centos-7.sh
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7 bash -c "
+ df -a
+ cd /work
+ export CCACHE_DIR=/work/.ccache
+ bash dev/ci-velox-buildstatic-centos-7.sh
+ ccache -s
+ mkdir -p /work/.m2/repository/org/apache/arrow/
+ cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
+ "
- name: Upload native libs
uses: actions/upload-artifact@v3
with:
@@ -65,10 +70,10 @@ jobs:
path: /root/.m2/repository/org/apache/arrow/
name: velox-arrow-jar-centos-7-${{github.sha}}
- build-bundle-package-centos7:
+ build-bundle-package-centos8:
needs: build-native-lib
runs-on: ubuntu-20.04
- container: centos:7
+ container: centos:8
steps:
- uses: actions/checkout@v2
- name: Download All Artifacts
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 56b43975ac..3c4bb3e2e9 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -164,12 +164,12 @@ jobs:
&& GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
--local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
- run-tpc-test-centos:
+ run-tpc-test-centos8:
needs: build-native-lib-centos-7
strategy:
fail-fast: false
matrix:
- os: [ "centos:8" ] #TODO: add centos7 tests back with docker run
+ os: [ "centos:8" ]
spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ]
java: [ "java-8", "java-11", "java-17" ]
# Spark supports JDK17 since 3.3 and later, see
https://issues.apache.org/jira/browse/SPARK-33772
@@ -255,6 +255,68 @@ jobs:
--local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
--extra-conf=spark.gluten.ras.enabled=true
+ run-tpc-test-centos7:
+ needs: build-native-lib-centos-7
+ strategy:
+ fail-fast: false
+ matrix:
+ spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ]
+ java: [ "java-8" ]
+ # Spark supports JDK17 since 3.3 and later, see
https://issues.apache.org/jira/browse/SPARK-33772
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Native Artifacts
+ uses: actions/download-artifact@v3
+ with:
+ name: velox-native-lib-centos-7-${{github.sha}}
+ path: ./cpp/build/releases/
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v3
+ with:
+ name: arrow-jars-centos-7-${{github.sha}}
+ path: .m2/repository/org/apache/arrow/
+ - name: Build and run TPCH/DS tests
+ run: |
+ docker pull centos:7
+ docker run -v $GITHUB_WORKSPACE:/work -v
/$GITHUB_WORKSPACE/.m2:/root/.m2/ -w /work \
+ -e matrix.java=${{ matrix.java }} -e matrix.spark=${{ matrix.spark
}} \
+ centos:7 \
+ bash -c "
+ sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-*
|| true
+ sed -i -e
's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* || true
+
+ # Setup java and maven
+ yum update -y && yum install -y java-1.8.0-openjdk-devel wget
+
+ wget -nv
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+ tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8
/usr/lib/maven
+ export PATH=${PATH}:/usr/lib/maven/bin
+
+ # Set environment variables
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+
+ # Build gluten-it
+ mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
+ cd /work/tools/gluten-it
+ mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }}
+
+ # Run TPC-H / TPC-DS
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
+
+ # Run TPC-H / TPC-DS with RAS
+ cd /work/tools/gluten-it
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
+ --extra-conf=spark.gluten.ras.enabled=true \
+ && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
+ --extra-conf=spark.gluten.ras.enabled=true
+ "
+
run-tpc-test-ubuntu-oom:
needs: build-native-lib-centos-7
strategy:
@@ -962,12 +1024,12 @@ jobs:
df -a
bash dev/ci-velox-buildshared-centos-8.sh
ccache -s
- - name: "Save ccache"
- uses: actions/cache/save@v3
- id: ccache
- with:
- path: '${{ env.CCACHE_DIR }}'
- key: ccache-centos8-release-default-${{github.sha}}
+ # - name: "Save ccache"
+ # uses: actions/cache/save@v3
+ # id: ccache
+ # with:
+ # path: '${{ env.CCACHE_DIR }}'
+ # key: ccache-centos8-release-default-${{github.sha}}
- name: Run CPP unit test
run: |
cd ./cpp/build && ctest -V
diff --git a/.github/workflows/velox_backend_cache.yml
b/.github/workflows/velox_backend_cache.yml
index e969ffb741..d0b295b5f4 100644
--- a/.github/workflows/velox_backend_cache.yml
+++ b/.github/workflows/velox_backend_cache.yml
@@ -31,9 +31,8 @@ concurrency:
jobs:
cache-native-lib-centos-7:
runs-on: ubuntu-20.04
- container: apache/gluten:vcpkg-centos-7
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v4
- name: Get Ccache
uses: actions/cache/restore@v3
with:
@@ -43,8 +42,9 @@ jobs:
ccache-centos7-release-default
- name: Build Gluten native libraries
run: |
- df -a
- bash dev/ci-velox-buildstatic-centos-7.sh
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7 bash -c "
+ bash dev/ci-velox-buildstatic-centos-7.sh
+ "
- name: Save Ccache
uses: actions/cache/save@v3
id: ccache
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
