This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 41ee44caac [VL] Adding nightly package for enhanced features (#11528)
41ee44caac is described below
commit 41ee44caac4052c7e57119066fc6430111f0b416
Author: Yuan <[email protected]>
AuthorDate: Wed Mar 4 12:02:13 2026 +0000
[VL] Adding nightly package for enhanced features (#11528)
* [VL] Adding nightly package for enhanced features
Only covered JDK17 here to reduce the package size in apache nightly
Signed-off-by: Yuan <[email protected]>
* add spark-40 also
Signed-off-by: Yuan <[email protected]>
---------
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/velox_nightly.yml | 90 ++++++++++++++++++++++++++++++++++++-
1 file changed, 89 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/velox_nightly.yml
b/.github/workflows/velox_nightly.yml
index 2b45b40f0f..2a8bf47994 100644
--- a/.github/workflows/velox_nightly.yml
+++ b/.github/workflows/velox_nightly.yml
@@ -287,6 +287,8 @@ jobs:
./build/mvn clean install -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ # iceberg support for spark-4.1 with java-17 is not avaiable, skip
it first
+ ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
@@ -321,7 +323,8 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/ && \
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
- ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ # iceberg support for spark-4.1 with java-21 is not avaiable, skip
it first
+ ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
@@ -416,3 +419,88 @@ jobs:
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+
+
+ # Build velox with enhanced features
+ build-native-lib-enhanced:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v4
+ - name: Get Ccache
+ uses: actions/cache/restore@v4
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-enhanced-centos7-release-default-${{github.sha}}
+ restore-keys: |
+ ccache-enhanced-centos7-release-default
+ - name: Build Gluten velox third party
+ run: |
+ docker run -v $GITHUB_WORKSPACE:/work -w /work
apache/gluten:vcpkg-centos-7 bash -c "
+ df -a
+ cd /work
+ export CCACHE_DIR=/work/.ccache
+ bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
+ ccache -s
+ mkdir -p /work/.m2/repository/org/apache/arrow/
+ cp -r /root/.m2/repository/org/apache/arrow/*
/work/.m2/repository/org/apache/arrow/
+ "
+ - name: Upload native libs
+ uses: actions/upload-artifact@v4
+ with:
+ path: ./cpp/build/releases/
+ name: velox-native-lib-enhanced-${{github.sha}}
+ retention-days: 1
+ - name: Upload Artifact Arrow Jar
+ uses: actions/upload-artifact@v4
+ with:
+ path: .m2/repository/org/apache/arrow/
+ name: velox-arrow-jar-centos-7-enhanced-${{github.sha}}
+
+ build-bundle-enhanced-package-centos8-jdk17:
+ needs: build-native-lib-enhanced
+ runs-on: ubuntu-22.04
+ container: centos:8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-enhanced-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-7-enhanced-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Setup java and maven
+ run: |
+ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
+ sed -i
's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* && \
+ yum update -y && yum install -y java-17-openjdk-devel wget
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Build package for Spark
+ run: |
+ cd $GITHUB_WORKSPACE/ && \
+ ./build/mvn clean install -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
+ ./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ # iceberg support for spark-4.1 with java-17 is not avaiable, skip
it first
+ ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
+ - name: Upload bundle package
+ uses: actions/upload-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk17-${{
steps.date.outputs.date }}
+ path: package/target/gluten-velox-bundle-*.jar
+ retention-days: 7
+ - name: rsync to apache nightly
+ uses:
burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
+ with:
+ switches: -avzr
+ path: package/target/gluten-velox-bundle-*.jar
+ remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH
}}/gluten/nightly-release-jdk17-enhanced
+ remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+ remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+ remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
