PHILO-HE commented on code in PR #9888:
URL: https://github.com/apache/incubator-gluten/pull/9888#discussion_r2137759675
##########
.github/workflows/velox_nightly.yml:
##########
@@ -92,17 +92,79 @@ jobs:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- - name: Build for Spark 3.5
+ - name: Build package for Spark
run: |
cd $GITHUB_WORKSPACE/ && \
export MAVEN_HOME=/usr/lib/maven && \
export PATH=${PATH}:${MAVEN_HOME}/bin && \
+ mvn clean install -Pspark-3.2 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
mvn clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
- name: nightly-gluten-velox-bundle-package-spark35-${{
steps.date.outputs.date }}
+ name: nightly-gluten-velox-bundle-package-jdk8-${{
steps.date.outputs.date }}
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
+ - name: rsync to apache nightly
+ uses: burnett01/[email protected]
+ with:
+ switches: -avzr
+ path: package/target/gluten-velox-bundle-*.jar
+ remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH
}}/gluten/nightly-release-jdk8
+ remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+ remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+ remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+ build-bundle-package-centos8-jdk17:
+ needs: build-native-lib
+ runs-on: ubuntu-22.04
+ container: centos:8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-7-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Setup java and maven
+ run: |
+ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
+ sed -i
's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* && \
+ yum update -y && yum install -y java-17-openjdk-devel wget && \
+ wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
&& \
+ tar -xvf apache-maven-3.8.8-bin.tar.gz && \
+ mv apache-maven-3.8.8 /usr/lib/maven
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Build package for Spark
+ run: |
+ cd $GITHUB_WORKSPACE/ && \
+ export MAVEN_HOME=/usr/lib/maven && \
+ export PATH=${PATH}:${MAVEN_HOME}/bin && \
+ mvn clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
Review Comment:
Do we need to add `-Pjava-17`？
##########
.github/workflows/velox_nightly.yml:
##########
@@ -92,17 +92,79 @@ jobs:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
- - name: Build for Spark 3.5
+ - name: Build package for Spark
run: |
cd $GITHUB_WORKSPACE/ && \
export MAVEN_HOME=/usr/lib/maven && \
export PATH=${PATH}:${MAVEN_HOME}/bin && \
+ mvn clean install -Pspark-3.2 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
mvn clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
- name: nightly-gluten-velox-bundle-package-spark35-${{
steps.date.outputs.date }}
+ name: nightly-gluten-velox-bundle-package-jdk8-${{
steps.date.outputs.date }}
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
+ - name: rsync to apache nightly
+ uses: burnett01/[email protected]
+ with:
+ switches: -avzr
+ path: package/target/gluten-velox-bundle-*.jar
+ remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH
}}/gluten/nightly-release-jdk8
+ remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+ remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+ remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+ build-bundle-package-centos8-jdk17:
+ needs: build-native-lib
+ runs-on: ubuntu-22.04
+ container: centos:8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-7-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Setup java and maven
+ run: |
+ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
+ sed -i
's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* && \
+ yum update -y && yum install -y java-17-openjdk-devel wget && \
+ wget
https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
&& \
+ tar -xvf apache-maven-3.8.8-bin.tar.gz && \
+ mv apache-maven-3.8.8 /usr/lib/maven
Review Comment:
This can be simplified a bit by running $SETUP install_maven like
velox_backend.yml. Ditto for the above job.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
