This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 7e6485743 fix: Re-enable Spark 4 tests on Linux (#1806)
7e6485743 is described below
commit 7e64857435d9b7f2cdfb58e950b4fce7f855fc00
Author: Andy Grove <agr...@apache.org>
AuthorDate: Wed May 28 12:19:33 2025 -0600
fix: Re-enable Spark 4 tests on Linux (#1806)
---
.github/actions/java-test/action.yaml | 14 +++++-
.github/workflows/pr_build_linux_spark4.yml | 74 ++++++++++++++++++++++-------
2 files changed, 71 insertions(+), 17 deletions(-)
diff --git a/.github/actions/java-test/action.yaml
b/.github/actions/java-test/action.yaml
index 53775da0d..c0ff974aa 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -18,6 +18,10 @@
name: "Java Test"
description: "Run Java tests"
inputs:
+ suites:
+ description: 'Which test suites to run'
+ required: false
+ default: ''
maven_opts:
description: 'Maven options passed to the mvn command'
required: false
@@ -51,10 +55,18 @@ runs:
run: |
./mvnw -B compile test-compile scalafix:scalafix -Dscalafix.mode=CHECK
-Psemanticdb ${{ inputs.maven_opts }}
- - name: Run tests
+ - name: Run all tests
shell: bash
+ if: ${{ inputs.suites == '' }}
run: |
MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError
-XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log"
SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }}
+ - name: Run specified tests
+ shell: bash
+ if: ${{ inputs.suites != '' }}
+ run: |
+ MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)"
+ echo "Running with MAVEN_SUITES=$MAVEN_SUITES"
+ MAVEN_OPTS="-DwildcardSuites=$MAVEN_SUITES
-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError
-XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log"
SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }}
- name: Upload crash logs
if: failure()
uses: actions/upload-artifact@v4
diff --git a/.github/workflows/pr_build_linux_spark4.yml
b/.github/workflows/pr_build_linux_spark4.yml
index dc0a96c8d..0c143eb53 100644
--- a/.github/workflows/pr_build_linux_spark4.yml
+++ b/.github/workflows/pr_build_linux_spark4.yml
@@ -23,17 +23,16 @@ concurrency:
on:
-# temporarily disabled on PRs due to
https://github.com/apache/datafusion-comet/issues/1786
-# push:
-# paths-ignore:
-# - "doc/**"
-# - "docs/**"
-# - "**.md"
-# pull_request:
-# paths-ignore:
-# - "doc/**"
-# - "docs/**"
-# - "**.md"
+ push:
+ paths-ignore:
+ - "doc/**"
+ - "docs/**"
+ - "**.md"
+ pull_request:
+ paths-ignore:
+ - "doc/**"
+ - "docs/**"
+ - "**.md"
# manual trigger
#
https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
workflow_dispatch:
@@ -50,10 +49,53 @@ jobs:
java_version: [17]
test-target: [java]
spark-version: ['4.0']
- is_push_event:
- - ${{ github.event_name == 'push' }}
+ suites:
+ - name: "fuzz"
+ value: |
+ org.apache.comet.CometFuzzTestSuite
+ org.apache.comet.DataGeneratorSuite
+ - name: "shuffle"
+ value: |
+ org.apache.comet.exec.CometShuffleSuite
+ org.apache.comet.exec.CometShuffle4_0Suite
+ org.apache.comet.exec.CometNativeShuffleSuite
+ org.apache.comet.exec.CometShuffleEncryptionSuite
+ org.apache.comet.exec.CometShuffleManagerSuite
+ org.apache.comet.exec.CometAsyncShuffleSuite
+ org.apache.comet.exec.DisableAQECometShuffleSuite
+ org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
+ - name: "parquet"
+ value: |
+ org.apache.comet.parquet.ParquetReadV1Suite
+ org.apache.comet.parquet.ParquetReadV2Suite
+ org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
+ org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
+ org.apache.spark.sql.comet.ParquetEncryptionITCase
+ org.apache.comet.exec.CometNativeReaderSuite
+ - name: "exec"
+ value: |
+ org.apache.comet.exec.CometAggregateSuite
+ org.apache.comet.exec.CometExec3_4PlusSuite
+ org.apache.comet.exec.CometExecSuite
+ org.apache.comet.exec.CometJoinSuite
+ org.apache.comet.CometArrayExpressionSuite
+ org.apache.comet.CometCastSuite
+ org.apache.comet.CometExpressionSuite
+ org.apache.comet.CometExpressionCoverageSuite
+ org.apache.comet.CometNativeSuite
+ org.apache.comet.CometSparkSessionExtensionsSuite
+ org.apache.spark.CometPluginsSuite
+ org.apache.spark.CometPluginsDefaultSuite
+ org.apache.spark.CometPluginsNonOverrideSuite
+ org.apache.spark.CometPluginsUnifiedModeOverrideSuite
+ org.apache.spark.sql.CometTPCDSQuerySuite
+ org.apache.spark.sql.CometTPCDSQueryTestSuite
+ org.apache.spark.sql.CometTPCHQuerySuite
+ org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite
+ org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite
+ org.apache.spark.sql.comet.CometTaskMetricsSuite
fail-fast: false
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
+ name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{
matrix.suites.name }}]
runs-on: ${{ matrix.os }}
container:
image: amd64/rust
@@ -67,6 +109,6 @@ jobs:
- name: Java test steps
uses: ./.github/actions/java-test
with:
- maven_opts: -Pspark-${{ matrix.spark-version }}
+ suites: ${{ matrix.suites.value }}
+ maven_opts: "-Pspark-${{ matrix.spark-version }}"
upload-test-reports: true
-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
