This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ea3ef5d966 [GLUTEN-11316][VL] Add Spark 4.1 with JDK 21 to nightly
builds (#11442)
ea3ef5d966 is described below
commit ea3ef5d966c6e5603a7a6940322b603fe621085f
Author: Mariam AlMesfer <[email protected]>
AuthorDate: Thu Jan 22 17:58:38 2026 +0300
[GLUTEN-11316][VL] Add Spark 4.1 with JDK 21 to nightly builds (#11442)
This PR adds Spark 4.1 with JDK 21 support to the nightly build workflow
Related issue: #11316
Co-authored-by: Mariam-Almesfer <[email protected]>
---
.github/workflows/velox_nightly.yml | 106 +++++++++++++++++++++++++++++++++++-
1 file changed, 105 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/velox_nightly.yml
b/.github/workflows/velox_nightly.yml
index 64053ba726..f2022f0a75 100644
--- a/.github/workflows/velox_nightly.yml
+++ b/.github/workflows/velox_nightly.yml
@@ -151,6 +151,43 @@ jobs:
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
+ build-bundle-package-centos9-jdk21-x86:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ needs: build-native-lib-x86
+ runs-on: ubuntu-22.04
+ container: quay.io/centos/centos:stream9
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-7-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Setup java and maven
+ run: |
+ yum update -y && yum install -y java-21-openjdk-devel wget
+ $SETUP install_maven
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Build package for Spark
+ run: |
+ cd $GITHUB_WORKSPACE/ && \
+ export MAVEN_HOME=/usr/lib/maven && \
+ export PATH=${PATH}:${MAVEN_HOME}/bin && \
+ mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ - name: Upload bundle package
+ uses: actions/upload-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk21-x86-${{
steps.date.outputs.date }}
+ path: package/target/gluten-velox-bundle-*.jar
+ retention-days: 7
#build and package for arm64
build-native-lib-centos-8-arm64:
@@ -273,6 +310,44 @@ jobs:
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
+ build-bundle-package-centos9-jdk21-arm64:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ needs: build-native-lib-centos-8-arm64
+ runs-on: ubuntu-22.04-arm
+ container: quay.io/centos/centos:stream9
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Setup java and maven
+ run: |
+ yum update -y && yum install -y java-21-openjdk-devel wget
+ $SETUP install_maven
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Build package for Spark
+ run: |
+ cd $GITHUB_WORKSPACE/ && \
+ export MAVEN_HOME=/usr/lib/maven && \
+ export PATH=${PATH}:${MAVEN_HOME}/bin && \
+ mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ - name: Upload bundle package
+ uses: actions/upload-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk21-arm64-${{
steps.date.outputs.date }}
+ path: package/target/gluten-velox-bundle-*.jar
+ retention-days: 7
+
# upload package to nightly.apache.org
upload-jdk8-package:
if: ${{ startsWith(github.repository, 'apache/') }}
@@ -330,4 +405,33 @@ jobs:
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
- remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
\ No newline at end of file
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+
+ upload-jdk21-package:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ needs: [ build-bundle-package-centos9-jdk21-arm64,
build-bundle-package-centos9-jdk21-x86 ]
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Download ARM JDK21 Package Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk21-arm64-${{
steps.date.outputs.date }}
+ path: package/
+ - name: Download X86 JDK21 Package Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk21-x86-${{
steps.date.outputs.date }}
+ path: package/
+ - name: rsync to apache nightly
+ uses:
burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
+ with:
+ switches: -avzr --delete
+ path: package/gluten-velox-bundle-*.jar
+ remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH
}}/gluten/nightly-release-jdk21
+ remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+ remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+ remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
