This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 5e9d8b7015 [CI] Skip irrelevant jobs in Velox CI workflows based on
changed paths (#12667)
5e9d8b7015 is described below
commit 5e9d8b7015c347545d2c6d668b337914f852f6bb
Author: Yuan <[email protected]>
AuthorDate: Tue Aug 18 12:37:59 2026 +0100
[CI] Skip irrelevant jobs in Velox CI workflows based on changed paths
(#12667)
* [CI] Skip irrelevant jobs in Velox CI workflows based on changed paths
Add a `detect-changes` job (using dorny/paths-filter@v3) to the three
Velox CI workflows so that downstream jobs are skipped when none of their
relevant source paths were touched.
velox_backend_x86.yml – 23 jobs wired
velox_backend_enhanced.yml – 5 jobs wired
velox_backend_arm.yml – 4 jobs wired
Change-detection flags:
cpp – cpp/**, ep/build-velox/**, dev/**
java – all cross-version Java/Scala/Maven modules
shims33/34/35/40/41 – version-specific shim layer + gluten-ut/sparkXX
tools_it – tools/gluten-it/**
Skip rules:
build-native-lib, cpp-test-udf-test, build-cudf → only when cpp=true
tpc-test-* → cpp OR java OR tools_it
spark-test-sparkXX / sparkXX-slow → java OR shimsXX OR cpp
build-fast-build-test → java OR cpp
Safety: a gluten-core change sets java=true and therefore runs all
spark-test-sparkXX jobs. Only a change scoped entirely to one
shims/sparkXX subtree skips the other versions.
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/velox_backend_arm.yml | 35 +++++-
.github/workflows/velox_backend_enhanced.yml | 47 +++++++-
.github/workflows/velox_backend_x86.yml | 166 ++++++++++++++++++++++++---
3 files changed, 226 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/velox_backend_arm.yml
b/.github/workflows/velox_backend_arm.yml
index c4c5fa395b..b02c128102 100644
--- a/.github/workflows/velox_backend_arm.yml
+++ b/.github/workflows/velox_backend_arm.yml
@@ -50,7 +50,34 @@ concurrency:
cancel-in-progress: true
jobs:
+ # Detect which parts of the codebase were modified so downstream jobs can
+ # skip work that is irrelevant to the change.
+ detect-changes:
+ runs-on: ubuntu-22.04
+ outputs:
+ cpp: ${{ steps.filter.outputs.cpp }}
+ java: ${{ steps.filter.outputs.java }}
+ tools_it: ${{ steps.filter.outputs.tools_it }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - name: Detect changed paths
+ id: filter
+ run: |
+ BASE=${{ github.event.pull_request.base.sha }}
+ HEAD=${{ github.sha }}
+ changed=$(git diff --name-only "$BASE" "$HEAD")
+ echo "$changed"
+
+ match() { echo "$changed" | grep -qE "$1" && echo true || echo
false; }
+
+ echo "cpp=$(match '^(cpp/|ep/build-velox/|dev/)')"
>> $GITHUB_OUTPUT
+ echo "java=$(match
'^(\.github/workflows/|pom\.xml|backends-velox/|gluten-(uniffle|celeborn|ras|core|substrait|arrow|delta|iceberg|hudi|paimon)/|gluten-ut/(common/|test/|pom\.xml)|package/|build/mvn)')"
>> $GITHUB_OUTPUT
+ echo "tools_it=$(match '^tools/gluten-it/')"
>> $GITHUB_OUTPUT
+
build-native-lib-centos-8:
+ needs: detect-changes
runs-on: ubuntu-24.04-arm
container: apache/gluten:vcpkg-centos-9
steps:
@@ -103,7 +130,11 @@ jobs:
if-no-files-found: error
tpc-test-centos8:
- needs: build-native-lib-centos-8
+ needs: [detect-changes, build-native-lib-centos-8]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -146,6 +177,8 @@ jobs:
--local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
cpp-test-udf-test:
+ needs: detect-changes
+ if: needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-24.04-arm
container: apache/gluten:centos-9-jdk8
steps:
diff --git a/.github/workflows/velox_backend_enhanced.yml
b/.github/workflows/velox_backend_enhanced.yml
index 0a657e0491..ac0b1bb18c 100644
--- a/.github/workflows/velox_backend_enhanced.yml
+++ b/.github/workflows/velox_backend_enhanced.yml
@@ -51,7 +51,36 @@ concurrency:
cancel-in-progress: true
jobs:
+ # Detect which parts of the codebase were modified so downstream jobs can
+ # skip work that is irrelevant to the change.
+ detect-changes:
+ runs-on: ubuntu-22.04
+ outputs:
+ cpp: ${{ steps.filter.outputs.cpp }}
+ java: ${{ steps.filter.outputs.java }}
+ shims35: ${{ steps.filter.outputs.shims35 }}
+ shims40: ${{ steps.filter.outputs.shims40 }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - name: Detect changed paths
+ id: filter
+ run: |
+ BASE=${{ github.event.pull_request.base.sha }}
+ HEAD=${{ github.sha }}
+ changed=$(git diff --name-only "$BASE" "$HEAD")
+ echo "$changed"
+
+ match() { echo "$changed" | grep -qE "$1" && echo true || echo
false; }
+
+ echo "cpp=$(match '^(cpp/|ep/build-velox/|dev/)')"
>> $GITHUB_OUTPUT
+ echo "java=$(match
'^(\.github/workflows/|pom\.xml|backends-velox/|gluten-(uniffle|celeborn|ras|core|substrait|arrow|delta|iceberg|hudi)/|gluten-ut/(common/|test/|pom\.xml)|package/|build/mvn)')"
>> $GITHUB_OUTPUT
+ echo "shims35=$(match
'^(shims/(spark35|common)/|gluten-ut/spark35/)')" >>
$GITHUB_OUTPUT
+ echo "shims40=$(match
'^(shims/(spark40|common)/|gluten-ut/spark40/)')" >>
$GITHUB_OUTPUT
+
build-native-lib-centos-7:
+ needs: detect-changes
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v7
@@ -99,7 +128,11 @@ jobs:
if-no-files-found: error
spark-test-spark35:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims35 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
container: apache/gluten:centos-8-jdk8
steps:
@@ -152,7 +185,11 @@ jobs:
path: /tmp/tpch-approved-plan/**
spark-test-spark35-slow:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims35 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
container: apache/gluten:centos-8-jdk8
steps:
@@ -192,7 +229,11 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark40:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims40 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index a9ff3ac042..89d759e432 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -52,7 +52,57 @@ concurrency:
cancel-in-progress: true
jobs:
+ # Detect which parts of the codebase were modified so downstream jobs can
+ # skip work that is irrelevant to the change (e.g. a Spark-3.3-shim-only
change does
+ # not need to run spark-test-spark34/35/40/41; C++ changes still run
spark-test-*
+ # jobs to validate JNI/native integration).
+ #
+ # Output flags:
+ # cpp – C++ sources, Velox build scripts, or dev tooling changed
+ # java – any cross-version Java/Scala/Maven source changed
+ # shims33/34/35/40/41 – version-specific shim layer changed
+ # tools_it – tools/gluten-it changed
+ #
+ # A job runs when ANY of the relevant flags is true, so:
+ # • gluten-core change → java=true → all spark-test jobs run ✓
+ # • shims/spark35 only → shims35=true → only spark35 jobs run ✓
+ # • pure cpp change → cpp=true → only native/tpc jobs run ✓
+ detect-changes:
+ runs-on: ubuntu-22.04
+ outputs:
+ cpp: ${{ steps.filter.outputs.cpp }}
+ java: ${{ steps.filter.outputs.java }}
+ shims33: ${{ steps.filter.outputs.shims33 }}
+ shims34: ${{ steps.filter.outputs.shims34 }}
+ shims35: ${{ steps.filter.outputs.shims35 }}
+ shims40: ${{ steps.filter.outputs.shims40 }}
+ shims41: ${{ steps.filter.outputs.shims41 }}
+ tools_it: ${{ steps.filter.outputs.tools_it }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - name: Detect changed paths
+ id: filter
+ run: |
+ BASE=${{ github.event.pull_request.base.sha }}
+ HEAD=${{ github.sha }}
+ changed=$(git diff --name-only "$BASE" "$HEAD")
+ echo "$changed"
+
+ match() { echo "$changed" | grep -qE "$1" && echo true || echo
false; }
+
+ echo "cpp=$(match '^(cpp/|ep/build-velox/|dev/)')"
>> $GITHUB_OUTPUT
+ echo "java=$(match
'^(\.github/workflows/|pom\.xml|backends-velox/|gluten-(uniffle|celeborn|ras|core|substrait|arrow|delta|iceberg|hudi|paimon)/|gluten-ut/(common/|test/|pom\.xml)|package/|build/mvn)')"
>> $GITHUB_OUTPUT
+ echo "shims33=$(match
'^(shims/(spark33|common)/|gluten-ut/spark33/)')" >>
$GITHUB_OUTPUT
+ echo "shims34=$(match
'^(shims/(spark34|common)/|gluten-ut/spark34/)')" >>
$GITHUB_OUTPUT
+ echo "shims35=$(match
'^(shims/(spark35|common)/|gluten-ut/spark35/)')" >>
$GITHUB_OUTPUT
+ echo "shims40=$(match
'^(shims/(spark40|common)/|gluten-ut/spark40/)')" >>
$GITHUB_OUTPUT
+ echo "shims41=$(match
'^(shims/(spark41|common)/|gluten-ut/spark41/)')" >>
$GITHUB_OUTPUT
+ echo "tools_it=$(match '^tools/gluten-it/')"
>> $GITHUB_OUTPUT
+
build-native-lib-centos-7:
+ needs: detect-changes
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v7
@@ -99,7 +149,11 @@ jobs:
if-no-files-found: error
tpc-test-ubuntu:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -228,7 +282,11 @@ jobs:
fi
tpc-test-centos8:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -311,7 +369,11 @@ jobs:
--local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1
tpc-test-centos7:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -354,7 +416,11 @@ jobs:
"
tpc-test-ubuntu-oom:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -464,7 +530,11 @@ jobs:
-d=IO_THREADS:0,spark.gluten.sql.columnar.backend.velox.IOThreads=0
tpc-test-ubuntu-randomkill:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -512,7 +582,11 @@ jobs:
--data-gen=skip --random-kill-tasks --no-session-reuse
--shard=${{ matrix.shard }}/3
tpc-test-centos8-uniffle:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -558,7 +632,11 @@ jobs:
--local --preset=velox-with-uniffle --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
tpc-test-ubuntu-2204-celeborn:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.cpp == 'true' ||
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.tools_it == 'true'
strategy:
fail-fast: false
matrix:
@@ -631,7 +709,11 @@ jobs:
fi
spark-test-spark33:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims33 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
env:
SPARK_TESTING: true
@@ -684,7 +766,11 @@ jobs:
path: /tmp/tpch-approved-plan/**
spark-test-spark33-slow:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims33 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
env:
SPARK_TESTING: true
@@ -727,7 +813,11 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark34:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims34 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -794,7 +884,11 @@ jobs:
path: /tmp/tpch-approved-plan/**
spark-test-spark34-slow:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims34 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -842,7 +936,11 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark35:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims35 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -910,7 +1008,11 @@ jobs:
path: /tmp/tpch-approved-plan/**
spark-test-spark35-scala213:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims35 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -972,7 +1074,11 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark35-slow:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims35 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -1021,6 +1127,8 @@ jobs:
**/gluten-ut/**/core.*
cpp-test-udf-test:
+ needs: detect-changes
+ if: needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
container: apache/gluten:centos-9-jdk8
steps:
@@ -1138,6 +1246,8 @@ jobs:
path: "**/target/*.log"
build-cudf-centos-9:
+ needs: detect-changes
+ if: needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
steps:
- name: "node-cleanup" # by default the free runner does not have enough
disk space
@@ -1188,6 +1298,10 @@ jobs:
"
build-fast-build-test:
+ needs: detect-changes
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
container: apache/gluten:centos-8-jdk17
steps:
@@ -1198,7 +1312,11 @@ jobs:
$MVN_CMD clean test-compile -Pspark-4.0 -Pscala-2.13
-Pbackends-velox -Pspark-ut -Piceberg,iceberg-test,delta,paimon -Pfast-build
spark-test-spark40:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims40 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -1260,7 +1378,11 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark40-slow:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims40 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -1309,7 +1431,11 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark41:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims41 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@@ -1380,7 +1506,11 @@ jobs:
**/gluten-ut/**/core.*
spark-test-spark41-slow:
- needs: build-native-lib-centos-7
+ needs: [detect-changes, build-native-lib-centos-7]
+ if: >-
+ needs.detect-changes.outputs.java == 'true' ||
+ needs.detect-changes.outputs.shims41 == 'true' ||
+ needs.detect-changes.outputs.cpp == 'true'
runs-on: ubuntu-22.04
strategy:
fail-fast: false
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
