This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 908ae9a52 chore: Speed up "PR Builds" CI workflows (#1807)
908ae9a52 is described below
commit 908ae9a52b414a2025b2b15ac4933816f571d17a
Author: Andy Grove <agr...@apache.org>
AuthorDate: Thu May 29 10:30:12 2025 -0600
chore: Speed up "PR Builds" CI workflows (#1807)
---
.github/actions/java-test/action.yaml | 8 +
.github/workflows/pr_build.yml | 317 ---------------------
...r_build_linux_spark4.yml => pr_build_linux.yml} | 77 ++++-
...r_build_linux_spark4.yml => pr_build_macos.yml} | 28 +-
4 files changed, 85 insertions(+), 345 deletions(-)
diff --git a/.github/actions/java-test/action.yaml
b/.github/actions/java-test/action.yaml
index c0ff974aa..634c42a64 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -26,6 +26,10 @@ inputs:
description: 'Maven options passed to the mvn command'
required: false
default: ''
+ scan_impl:
+ description: 'The default Parquet scan implementation'
+ required: false
+ default: 'native_comet'
upload-test-reports:
description: 'Whether to upload test results including coverage to GitHub'
required: false
@@ -58,11 +62,15 @@ runs:
- name: Run all tests
shell: bash
if: ${{ inputs.suites == '' }}
+ env:
+ COMET_PARQUET_SCAN_IMPL: ${{ inputs.scan_impl }}
run: |
MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError
-XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log"
SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }}
- name: Run specified tests
shell: bash
if: ${{ inputs.suites != '' }}
+ env:
+ COMET_PARQUET_SCAN_IMPL: ${{ inputs.scan_impl }}
run: |
MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)"
echo "Running with MAVEN_SUITES=$MAVEN_SUITES"
diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
deleted file mode 100644
index 308dd2d68..000000000
--- a/.github/workflows/pr_build.yml
+++ /dev/null
@@ -1,317 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: PR Build
-
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
-
-on:
- push:
- paths-ignore:
- - "doc/**"
- - "docs/**"
- - "**.md"
- pull_request:
- paths-ignore:
- - "doc/**"
- - "docs/**"
- - "**.md"
- # manual trigger
- #
https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
- workflow_dispatch:
-
-env:
- RUST_VERSION: stable
-
-jobs:
-
- # run Comet test with full matrix of Java and Scala versions for the latest
supported 3.5.x release
- linux-test:
- strategy:
- matrix:
- os: [ubuntu-latest]
- java_version: [11, 17]
- test-target: [rust, java]
- spark-version: ['3.5']
- scala-version: ['2.12', '2.13']
- is_push_event:
- - ${{ github.event_name == 'push' }}
- exclude: # exclude java 11 for pull_request event
- - java_version: 11
- is_push_event: false
- fail-fast: false
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{
matrix.test-target }}
- runs-on: ${{ matrix.os }}
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- - if: matrix.test-target == 'rust'
- name: Rust test steps
- uses: ./.github/actions/rust-test
- - if: matrix.test-target == 'java'
- name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }}
- # upload test reports only for java 17
- upload-test-reports: ${{ matrix.java_version == '17' }}
-
- # run Comet tests for older 3.5.x releases with one java and scala versions
- linux-test-35-minor-versions:
- strategy:
- matrix:
- os: [ubuntu-latest]
- java_version: [17]
- test-target: [java]
- spark-version: ['3.5']
- spark-patch-version: ['3.5.2', '3.5.3', '3.5.4']
- scala-version: ['2.12']
- is_push_event:
- - ${{ github.event_name == 'push' }}
- fail-fast: false
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{
matrix.test-target }}
- runs-on: ${{ matrix.os }}
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- - if: matrix.test-target == 'rust'
- name: Rust test steps
- uses: ./.github/actions/rust-test
- - if: matrix.test-target == 'java'
- name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}"
- # upload test reports only for java 17
- upload-test-reports: ${{ matrix.java_version == '17' }}
-
- linux-test-native-datafusion-scan:
- strategy:
- matrix:
- os: [ubuntu-latest]
- scan_impl: ['native_datafusion', 'native_iceberg_compat']
- java_version: [17]
- test-target: [rust, java]
- spark-version: ['3.5']
- scala-version: ['2.12']
- is_push_event:
- - ${{ github.event_name == 'push' }}
- fail-fast: false
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{
matrix.test-target }}-${{ matrix.scan_impl }}
- runs-on: ${{ matrix.os }}
- env:
- COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }}
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- - if: matrix.test-target == 'rust'
- name: Rust test steps
- uses: ./.github/actions/rust-test
- - if: matrix.test-target == 'java'
- name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }}
- # upload test reports only for java 17
- upload-test-reports: ${{ matrix.java_version == '17' }}
-
- linux-test-with-old-spark:
- strategy:
- matrix:
- os: [ubuntu-latest]
- java_version: [11, 17]
- test-target: [java]
- spark-version: ['3.4']
- scala-version: ['2.12', '2.13']
- fail-fast: false
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{
matrix.test-target }}
- runs-on: ${{ matrix.os }}
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- - name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }}
-
- macos-test:
- strategy:
- matrix:
- os: [macos-13]
- java_version: [11, 17]
- test-target: [rust, java]
- spark-version: ['3.4', '3.5']
- scala-version: ['2.12', '2.13']
- fail-fast: false
- if: github.event_name == 'push'
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{
matrix.test-target }}
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-macos-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- - if: matrix.test-target == 'rust'
- name: Rust test steps
- uses: ./.github/actions/rust-test
- - if: matrix.test-target == 'java'
- name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }}
-
- macos-aarch64-test:
- strategy:
- matrix:
- java_version: [11, 17]
- test-target: [rust, java]
- spark-version: ['3.4', '3.5']
- scala-version: ['2.12', '2.13']
- is_push_event:
- - ${{ github.event_name == 'push' }}
- exclude: # exclude java 11 for pull_request event
- - java_version: 11
- is_push_event: false
- fail-fast: false
- name: macos-14(Silicon)/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{
matrix.test-target }}
- runs-on: macos-14
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-macos-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- jdk-architecture: aarch64
- protoc-architecture: aarch_64
- - if: matrix.test-target == 'rust'
- name: Rust test steps
- uses: ./.github/actions/rust-test
- - if: matrix.test-target == 'java'
- name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }}
-
- macos-test-with-spark4_0:
- strategy:
- matrix:
- os: [macos-13]
- java_version: [17]
- test-target: [java]
- spark-version: ['4.0']
- fail-fast: false
- if: github.event_name == 'push'
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-macos-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- - name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }}
- upload-test-reports: true
-
- macos-aarch64-test-with-spark4_0:
- strategy:
- matrix:
- java_version: [17]
- test-target: [java]
- spark-version: ['4.0']
- is_push_event:
- - ${{ github.event_name == 'push' }}
- exclude: # exclude java 11 for pull_request event
- - java_version: 11
- is_push_event: false
- fail-fast: false
- name: macos-14(Silicon)/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
- runs-on: macos-14
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-macos-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- jdk-architecture: aarch64
- protoc-architecture: aarch_64
- - name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }}
- upload-test-reports: true
-
- macos-aarch64-test-with-old-spark:
- strategy:
- matrix:
- java_version: [17]
- test-target: [java]
- spark-version: ['3.4']
- scala-version: ['2.12', '2.13']
- exclude:
- - java_version: 8
- fail-fast: false
- name: macos-14(Silicon)/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{
matrix.test-target }}
- runs-on: macos-14
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-macos-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
- jdk-architecture: aarch64
- protoc-architecture: aarch_64
- - if: matrix.test-target == 'java'
- name: Java test steps
- uses: ./.github/actions/java-test
- with:
- maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }}
-
diff --git a/.github/workflows/pr_build_linux_spark4.yml
b/.github/workflows/pr_build_linux.yml
similarity index 64%
copy from .github/workflows/pr_build_linux_spark4.yml
copy to .github/workflows/pr_build_linux.yml
index 0c143eb53..732b5b4d1 100644
--- a/.github/workflows/pr_build_linux_spark4.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -15,14 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-name: PR Build (Linux + Spark 4)
+name: PR Build (Linux)
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
cancel-in-progress: true
on:
-
push:
paths-ignore:
- "doc/**"
@@ -41,15 +40,65 @@ env:
RUST_VERSION: stable
jobs:
-
- linux-test-with-spark4_0:
+
+ # Run Rust tests once per JDK version
+ linux-test-rust:
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ java_version: [11, 17]
+ fail-fast: false
+ name: ${{ matrix.os }}/java ${{ matrix.java_version }}-rust
+ runs-on: ${{ matrix.os }}
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Rust & Java toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: ${{env.RUST_VERSION}}
+ jdk-version: ${{ matrix.java_version }}
+ - name: Rust test steps
+ uses: ./.github/actions/rust-test
+
+ linux-test:
strategy:
matrix:
os: [ubuntu-latest]
- java_version: [17]
- test-target: [java]
- spark-version: ['4.0']
- suites:
+ # the goal with these profiles is to get coverage of all Java, Scala,
and Spark
+ # versions without testing all possible combinations, which would be
overkill
+ profile:
+ - name: "Spark 3.4, JDK 11, Scala 2.12"
+ java_version: "11"
+ maven_opts: "-Pspark-3.4 -Pscala-2.12"
+ scan_impl: "native_comet"
+
+ - name: "Spark 3.5, JDK 11, Scala 2.12"
+ java_version: "17"
+ maven_opts: "-Pspark-3.5 -Pscala-2.12"
+ scan_impl: "native_comet"
+
+ - name: "Spark 3.5, JDK 17, Scala 2.13"
+ java_version: "17"
+ maven_opts: "-Pspark-3.5 -Pscala-2.13"
+ scan_impl: "native_comet"
+
+ - name: "Spark 3.5, JDK 17, Scala 2.12 native_datafusion"
+ java_version: "17"
+ maven_opts: "-Pspark-3.5 -Pscala-2.12"
+ scan_impl: "native_datafusion"
+
+ - name: "Spark 3.5, JDK 17, Scala 2.12 native_iceberg_compat"
+ java_version: "17"
+ maven_opts: "-Pspark-3.5 -Pscala-2.12"
+ scan_impl: "native_iceberg_compat"
+
+ - name: "Spark 4.0, JDK 17"
+ java_version: "17"
+ maven_opts: "-Pspark-4.0"
+ scan_impl: "native_comet"
+ suite:
- name: "fuzz"
value: |
org.apache.comet.CometFuzzTestSuite
@@ -84,6 +133,7 @@ jobs:
org.apache.comet.CometExpressionCoverageSuite
org.apache.comet.CometNativeSuite
org.apache.comet.CometSparkSessionExtensionsSuite
+ org.apache.comet.CometStringExpressionSuite
org.apache.spark.CometPluginsSuite
org.apache.spark.CometPluginsDefaultSuite
org.apache.spark.CometPluginsNonOverrideSuite
@@ -95,7 +145,7 @@ jobs:
org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite
org.apache.spark.sql.comet.CometTaskMetricsSuite
fail-fast: false
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{
matrix.suites.name }}]
+ name: ${{ matrix.os }}/${{ matrix.profile.name }} [${{ matrix.suite.name
}}]
runs-on: ${{ matrix.os }}
container:
image: amd64/rust
@@ -105,10 +155,11 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{ matrix.java_version }}
+ jdk-version: ${{ matrix.profile.java_version }}
- name: Java test steps
uses: ./.github/actions/java-test
with:
- suites: ${{ matrix.suites.value }}
- maven_opts: "-Pspark-${{ matrix.spark-version }}"
- upload-test-reports: true
+ suites: ${{ matrix.suite.value }}
+ maven_opts: ${{ matrix.profile.maven_opts }}
+ scan_impl: ${{ matrix.profile.scan_impl }}
+ upload-test-reports: true
\ No newline at end of file
diff --git a/.github/workflows/pr_build_linux_spark4.yml
b/.github/workflows/pr_build_macos.yml
similarity index 86%
rename from .github/workflows/pr_build_linux_spark4.yml
rename to .github/workflows/pr_build_macos.yml
index 0c143eb53..85b40ac5b 100644
--- a/.github/workflows/pr_build_linux_spark4.yml
+++ b/.github/workflows/pr_build_macos.yml
@@ -15,14 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-name: PR Build (Linux + Spark 4)
+name: PR Build (macOS)
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
cancel-in-progress: true
on:
-
push:
paths-ignore:
- "doc/**"
@@ -42,14 +41,13 @@ env:
jobs:
- linux-test-with-spark4_0:
+ macos-aarch64-test:
strategy:
matrix:
- os: [ubuntu-latest]
java_version: [17]
- test-target: [java]
- spark-version: ['4.0']
- suites:
+ spark-version: ['3.4', '3.5', '4.0']
+ scala-version: ['2.13']
+ suite:
- name: "fuzz"
value: |
org.apache.comet.CometFuzzTestSuite
@@ -84,6 +82,7 @@ jobs:
org.apache.comet.CometExpressionCoverageSuite
org.apache.comet.CometNativeSuite
org.apache.comet.CometSparkSessionExtensionsSuite
+ org.apache.comet.CometStringExpressionSuite
org.apache.spark.CometPluginsSuite
org.apache.spark.CometPluginsDefaultSuite
org.apache.spark.CometPluginsNonOverrideSuite
@@ -95,20 +94,19 @@ jobs:
org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite
org.apache.spark.sql.comet.CometTaskMetricsSuite
fail-fast: false
- name: ${{ matrix.os }}/java ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{
matrix.suites.name }}]
- runs-on: ${{ matrix.os }}
- container:
- image: amd64/rust
+ name: macos-14(Silicon)/ ${{ matrix.java_version
}}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}
[${{matrix.suite.name}}]
+ runs-on: macos-14
steps:
- uses: actions/checkout@v4
- name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-builder
+ uses: ./.github/actions/setup-macos-builder
with:
rust-version: ${{env.RUST_VERSION}}
jdk-version: ${{ matrix.java_version }}
+ jdk-architecture: aarch64
+ protoc-architecture: aarch_64
- name: Java test steps
uses: ./.github/actions/java-test
with:
- suites: ${{ matrix.suites.value }}
- maven_opts: "-Pspark-${{ matrix.spark-version }}"
- upload-test-reports: true
+ suites: ${{ matrix.suite.value }}
+ maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{
matrix.scala-version }}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
