This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d73f608391 [MINOR] Remove redundant Spark resource setup from CI
(#12715)
d73f608391 is described below
commit d73f608391dfb714476fa06cc4f04392dd90448e
Author: Wechar Yu <[email protected]>
AuthorDate: Fri Aug 7 07:58:14 2026 +0800
[MINOR] Remove redundant Spark resource setup from CI (#12715)
---
.github/workflows/velox_backend_ansi.yml | 16 -----------
.github/workflows/velox_backend_enhanced.yml | 11 --------
.github/workflows/velox_backend_x86.yml | 41 ----------------------------
3 files changed, 68 deletions(-)
diff --git a/.github/workflows/velox_backend_ansi.yml
b/.github/workflows/velox_backend_ansi.yml
index 421d8485d8..5f4bb1d524 100644
--- a/.github/workflows/velox_backend_ansi.yml
+++ b/.github/workflows/velox_backend_ansi.yml
@@ -160,10 +160,6 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Prepare Spark Resources
- run: |
- rm -rf /opt/shims/spark41
- bash .github/workflows/util/install-spark-resources.sh 4.1
- name: "Spark 4.1 backends-velox Tests (ANSI ON)"
run: |
set -o pipefail
@@ -232,10 +228,6 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Prepare Spark Resources
- run: |
- rm -rf /opt/shims/spark41
- bash .github/workflows/util/install-spark-resources.sh 4.1
- name: "Spark 4.1 spark-ut Tests (ANSI ON)"
run: |
set -o pipefail
@@ -314,10 +306,6 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Prepare Spark Resources
- run: |
- rm -rf /opt/shims/spark40
- bash .github/workflows/util/install-spark-resources.sh 4.0
- name: "Spark 4.0 backends-velox Tests (ANSI ON)"
run: |
set -o pipefail
@@ -386,10 +374,6 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Prepare Spark Resources
- run: |
- rm -rf /opt/shims/spark40
- bash .github/workflows/util/install-spark-resources.sh 4.0
- name: "Spark 4.0 spark-ut Tests (ANSI ON)"
run: |
set -o pipefail
diff --git a/.github/workflows/velox_backend_enhanced.yml
b/.github/workflows/velox_backend_enhanced.yml
index fe36e8856c..408244b8ea 100644
--- a/.github/workflows/velox_backend_enhanced.yml
+++ b/.github/workflows/velox_backend_enhanced.yml
@@ -153,9 +153,6 @@ jobs:
spark-test-spark35-slow:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
- env: #TODO remove after image update
- LANG: C.UTF-8
- LC_ALL: C.UTF-8
container: apache/gluten:centos-8-jdk8
steps:
- uses: actions/checkout@v7
@@ -164,10 +161,6 @@ jobs:
with:
name: velox-native-lib-enhanced-centos-7-${{github.sha}}
path: ./cpp/build/releases
- - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark35
- bash .github/workflows/util/install-spark-resources.sh 3.5
- name: Build and Run unit test for Spark 3.5.5 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
@@ -221,10 +214,6 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Prepare Spark Resources for Spark 4.0.1 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark40
- bash .github/workflows/util/install-spark-resources.sh 4.0
- name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other
tests, group ${{ matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 2e34f3084b..577fcc3a1a 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -19,8 +19,6 @@ on:
pull_request:
paths:
- '.github/workflows/velox_backend_x86.yml'
- - '.github/workflows/util/install-spark-deps.sh' #TODO remove after
image update
- - '.github/workflows/util/install-spark-resources.sh' #TODO remove after
image update
- 'pom.xml'
- 'backends-velox/**'
- 'gluten-uniffle/**'
@@ -691,8 +689,6 @@ jobs:
runs-on: ubuntu-22.04
env:
SPARK_TESTING: true
- LANG: C.UTF-8 #TODO remove after image update
- LC_ALL: C.UTF-8
container: apache/gluten:centos-8-jdk8
steps:
- uses: actions/checkout@v7
@@ -701,10 +697,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Prepare Spark Resources for Spark 3.3.1 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark33
- bash .github/workflows/util/install-spark-resources.sh 3.3
- name: Build and Run unit test for Spark 3.3.1 (slow tests)
run: |
@@ -809,9 +801,6 @@ jobs:
fail-fast: false
matrix:
suite: [extended, slow-hive]
- env: #TODO remove after image update
- LANG: C.UTF-8
- LC_ALL: C.UTF-8
container: apache/gluten:centos-8-jdk8
steps:
- uses: actions/checkout@v7
@@ -820,10 +809,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Prepare Spark Resources for Spark 3.4.4 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark34
- bash .github/workflows/util/install-spark-resources.sh 3.4
- name: Build and Run unit test for Spark 3.4.4 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
@@ -996,8 +981,6 @@ jobs:
suite: [extended, slow-hive]
env:
SPARK_TESTING: true
- LANG: C.UTF-8 #TODO remove after image update
- LC_ALL: C.UTF-8
container: apache/gluten:centos-8-jdk8
steps:
- uses: actions/checkout@v7
@@ -1006,10 +989,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark35
- bash .github/workflows/util/install-spark-resources.sh 3.5
- name: Build and Run unit test for Spark 3.5.5 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
@@ -1209,10 +1188,6 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Prepare Spark Resources for Spark 4.0.2 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark40
- bash .github/workflows/util/install-spark-resources.sh 4.0
- name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other
tests, group ${{ matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
@@ -1260,8 +1235,6 @@ jobs:
suite: [extended, slow-hive]
env:
SPARK_TESTING: true
- LANG: C.UTF-8 #TODO remove after image update
- LC_ALL: C.UTF-8
container: apache/gluten:centos-8-jdk17
steps:
- uses: actions/checkout@v7
@@ -1270,10 +1243,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Prepare Spark Resources for Spark 4.0.2 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark40
- bash .github/workflows/util/install-spark-resources.sh 4.0
- name: Build and Run unit test for Spark 4.0 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
@@ -1336,10 +1305,6 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Prepare Spark Resources for Spark 4.1.1 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark41
- bash .github/workflows/util/install-spark-resources.sh 4.1
- name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other
tests, group ${{ matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
@@ -1391,8 +1356,6 @@ jobs:
suite: [extended, slow-hive]
env:
SPARK_TESTING: true
- LANG: C.UTF-8 #TODO remove after image update
- LC_ALL: C.UTF-8
container: apache/gluten:centos-9-jdk17
steps:
- uses: actions/checkout@v7
@@ -1401,10 +1364,6 @@ jobs:
with:
name: velox-native-lib-centos-7-${{github.sha}}
path: ./cpp/build/
- - name: Prepare Spark Resources for Spark 4.1.1 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark41
- bash .github/workflows/util/install-spark-resources.sh 4.1
- name: Build and Run unit test for Spark 4.1 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
