zhouyuan commented on code in PR #9209:
URL: https://github.com/apache/incubator-gluten/pull/9209#discussion_r2025273421
##########
.github/workflows/velox_backend.yml:
##########
@@ -908,116 +863,29 @@ jobs:
- name: Build and Run unit test for Spark 3.4.4 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- export SPARK_HOME=/opt/shims/spark34/spark_home/
yum install -y java-17-openjdk-devel
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- ls -l /opt/shims/spark34/spark_home/
- $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Pspark-ut -Phudi \
- -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
- -DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/
${EXTRA_FLAGS}"
- - name: Upload test report
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: test-report-spark34-slow-jdk17
- path: '**/surefire-reports/TEST-*.xml'
- - name: Upload unit tests log files
- if: ${{ !success() }}
- uses: actions/upload-artifact@v4
- with:
- name: ${{ github.job }}-test-log
- path: "**/target/*.log"
-
- run-spark-test-spark34-slow-jdk8:
- needs: build-native-lib-centos-7
- runs-on: ubuntu-20.04
- container: apache/gluten:centos-8-jdk8
- steps:
- - uses: actions/checkout@v2
- - name: Download All Artifacts
- uses: actions/download-artifact@v4
- with:
- name: velox-native-lib-centos-7-${{github.sha}}
- path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- - name: Build and Run unit test for Spark 3.4.4 (slow tests)
- run: |
- cd $GITHUB_WORKSPACE/
- export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
export SPARK_HOME=/opt/shims/spark34/spark_home/
- ls -l /opt/shims/spark34/spark_home/
- $MVN_CMD clean test -Pspark-3.4 -Pjava-8 -Pbackends-velox -Pdelta
-Pspark-ut -Phudi \
+ ls -l $SPARK_HOME
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Pspark-ut -Phudi \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
- -DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/"
- - name: Upload test report
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: test-report-spark34-slow-jdk8
- path: '**/surefire-reports/TEST-*.xml'
- - name: Upload unit tests log files
- if: ${{ !success() }}
- uses: actions/upload-artifact@v4
- with:
- name: ${{ github.job }}-test-log
- path: "**/target/*.log"
-
- run-spark-test-spark35:
- needs: build-native-lib-centos-7
- runs-on: ubuntu-20.04
- container: apache/gluten:centos-8-jdk8
- steps:
- - uses: actions/checkout@v2
- - name: Download All Artifacts
- uses: actions/download-artifact@v4
- with:
- name: velox-native-lib-centos-7-${{github.sha}}
- path: ./cpp/build/releases
- - name: Download Arrow Jars
- uses: actions/download-artifact@v4
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /root/.m2/repository/org/apache/arrow/
- - name: Prepare
- run: |
- dnf module -y install python39 && \
- alternatives --set python3 /usr/bin/python3.9 && \
- pip3 install setuptools==77.0.3 && \
- pip3 install pyspark==3.5.2 cython && \
- pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.2 (other tests)
- run: |
- cd $GITHUB_WORKSPACE/
- export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta
-Phudi -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+ -DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}"
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: test-report-spark35
+ name: ${{ github.job }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-test-log
path: "**/target/*.log"
- - name: Upload golden files
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: golden-files-spark35
- path: /tmp/tpch-approved-plan/**
- run-spark-test-spark35-jdk17:
+ spark-test-spark35:
needs: build-native-lib-centos-7
runs-on: ubuntu-20.04
container: apache/gluten:centos-8-jdk8
Review Comment:
should change with jdk17 image for this job also
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
