This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new a7b2fa912f [#10155] improvement(CI): Accelerate CI pipeline (#10156)
a7b2fa912f is described below
commit a7b2fa912f3f7541de3f463baf244a3ca0eb4cc1
Author: Qi Yu <[email protected]>
AuthorDate: Wed Mar 4 19:22:46 2026 +0800
[#10155] improvement(CI): Accelerate CI pipeline (#10156)
### What changes were proposed in this pull request?
This pull request makes significant improvements to the GitHub Actions
workflows for integration and build testing. The main changes focus on
more efficient test execution, better detection of changed modules, and
consistent naming and exclusion logic across workflows. The updates help
optimize CI resource usage and improve maintainability.
### Why are the changes needed?
For better development
Fix: #10155
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
CI
---
.../workflows/access-control-integration-test.yml | 2 +-
.../workflows/backend-integration-test-action.yml | 36 +++++++++++++++------
.github/workflows/backend-integration-test.yml | 37 +++++++++++++++++++++-
.github/workflows/build.yml | 21 ++++++++++--
.github/workflows/cron-integration-test.yml | 2 +-
.github/workflows/flink-integration-test.yml | 2 +-
.github/workflows/frontend-integration-test.yml | 19 +++++++----
.github/workflows/gvfs-fuse-build-test.yml | 2 +-
.github/workflows/python-integration-test.yml | 2 +-
.github/workflows/spark-integration-test.yml | 2 +-
.github/workflows/trino-integration-test.yml | 2 +-
11 files changed, 101 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/access-control-integration-test.yml
b/.github/workflows/access-control-integration-test.yml
index bfc5aeec59..ef6162bcd2 100644
--- a/.github/workflows/access-control-integration-test.yml
+++ b/.github/workflows/access-control-integration-test.yml
@@ -43,7 +43,7 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
diff --git a/.github/workflows/backend-integration-test-action.yml
b/.github/workflows/backend-integration-test-action.yml
index 3b1d82a843..08d054520f 100644
--- a/.github/workflows/backend-integration-test-action.yml
+++ b/.github/workflows/backend-integration-test-action.yml
@@ -20,6 +20,11 @@ on:
required: true
description: 'run on embedded or deploy mode'
type: string
+ changed-catalogs-contrib-modules:
+ required: false
+ description: 'space-separated changed catalogs-contrib module names,
or __all__'
+ type: string
+ default: ''
jobs:
start-runner:
@@ -47,7 +52,7 @@ jobs:
- name: Package Gravitino
if: ${{ inputs.test-mode == 'deploy' }}
run: |
- ./gradlew compileDistribution -x test
+ ./gradlew compileDistribution -x test -x web-v2:web:build
- name: Free up disk space
run: |
@@ -60,14 +65,27 @@ jobs:
- name: Backend Integration Test (JDK${{ inputs.java-version }}-${{
inputs.test-mode }}-${{ inputs.backend }})
id: integrationTest
- run: >
- ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }}
-PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false
- -x :web:web:test -x :web:integration-test:test -x :web-v2:web:test
-x :web-v2:integration-test:test -x :clients:client-python:test -x
:flink-connector:flink:test -x :spark-connector:spark-common:test
- -x :spark-connector:spark-3.3:test -x
:spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test
- -x :spark-connector:spark-runtime-3.3:test -x
:spark-connector:spark-runtime-3.4:test -x
:spark-connector:spark-runtime-3.5:test
- -x :trino-connector:integration-test:test -x
:trino-connector:trino-connector:test
- -x :authorizations:authorization-chain:test -x
:authorizations:authorization-ranger:test
- -x :catalogs-contrib:catalog-jdbc-oceanbase:test
+ run: |
+ ALL_CONTRIB_MODULES="$(find catalogs-contrib -mindepth 1 -maxdepth 1
-type d -name 'catalog-*' -exec basename {} \; | sort)"
+ CHANGED_CONTRIB_MODULES="${{ inputs.changed-catalogs-contrib-modules
}}"
+
+ EXCLUDE_CONTRIB_TESTS=""
+ if [ "$CHANGED_CONTRIB_MODULES" != "__all__" ]; then
+ for module in $ALL_CONTRIB_MODULES; do
+ if [[ " $CHANGED_CONTRIB_MODULES " != *" $module "* ]]; then
+ EXCLUDE_CONTRIB_TESTS="$EXCLUDE_CONTRIB_TESTS -x
:catalogs-contrib:$module:test"
+ fi
+ done
+ fi
+
+ ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }}
-PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false \
+ -x :web:web:test -x :web:integration-test:test -x :web-v2:web:test
-x :web-v2:integration-test:test -x :clients:client-python:test -x
:flink-connector:flink:test -x :spark-connector:spark-common:test \
+ -x :spark-connector:spark-3.3:test -x
:spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test \
+ -x :spark-connector:spark-runtime-3.3:test -x
:spark-connector:spark-runtime-3.4:test -x
:spark-connector:spark-runtime-3.5:test \
+ -x :trino-connector:integration-test:test -x
:trino-connector:trino-connector:test \
+ -x :authorizations:authorization-chain:test -x
:authorizations:authorization-ranger:test \
+ -x :clients:cli:test -x :maintenance:jobs:test -x
:maintenance:optimizer:test \
+ $EXCLUDE_CONTRIB_TESTS
- name: Upload integrate tests reports
uses: actions/upload-artifact@v4
diff --git a/.github/workflows/backend-integration-test.yml
b/.github/workflows/backend-integration-test.yml
index 376a3920d6..92cabf4fbc 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -16,6 +16,8 @@ jobs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
@@ -25,6 +27,7 @@ jobs:
- api/**
- bin/**
- catalogs/**
+ - catalogs-contrib/**
- clients/client-java/**
- clients/client-java-runtime/**
- clients/filesystem-hadoop3/**
@@ -45,9 +48,39 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
+ - name: Detect changed catalogs-contrib modules
+ id: contrib
+ run: |
+ if [ "${{ github.event_name }}" = "pull_request" ]; then
+ BASE_SHA="${{ github.event.pull_request.base.sha }}"
+ HEAD_SHA="${{ github.event.pull_request.head.sha }}"
+ else
+ BASE_SHA="${{ github.event.before }}"
+ HEAD_SHA="${{ github.sha }}"
+ fi
+
+ if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" =
"0000000000000000000000000000000000000000" ]; then
+ BASE_SHA="$(git rev-list --max-parents=0 "$HEAD_SHA" | tail -n 1)"
+ fi
+
+ MERGE_BASE_SHA="$(git merge-base "$BASE_SHA" "$HEAD_SHA" || true)"
+ if [ -z "$MERGE_BASE_SHA" ]; then
+ MERGE_BASE_SHA="$BASE_SHA"
+ fi
+
+ CHANGED_FILES="$(git diff --name-only "$MERGE_BASE_SHA" "$HEAD_SHA")"
+ CHANGED_MODULES="$(echo "$CHANGED_FILES" | awk -F/
'/^catalogs-contrib\/[^/]+\// {print $2}' | sort -u | xargs)"
+
+ # If shared catalogs-contrib build logic changes, run all contrib
catalog tests.
+ if echo "$CHANGED_FILES" | grep -q
'^catalogs-contrib/build.gradle.kts$'; then
+ CHANGED_MODULES="__all__"
+ fi
+
+ echo "changed_catalogs_contrib_modules=$CHANGED_MODULES" >>
"$GITHUB_OUTPUT"
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
+ changed_catalogs_contrib_modules: ${{
steps.contrib.outputs.changed_catalogs_contrib_modules }}
BackendIT-on-push:
needs: changes
@@ -71,6 +104,7 @@ jobs:
java-version: ${{ matrix.java-version }}
backend: ${{ matrix.backend }}
test-mode: ${{ matrix.test-mode }}
+ changed-catalogs-contrib-modules: ${{
needs.changes.outputs.changed_catalogs_contrib_modules }}
BackendIT-on-pr:
needs: changes
@@ -94,3 +128,4 @@ jobs:
java-version: ${{ matrix.java-version }}
backend: ${{ matrix.backend }}
test-mode: ${{ matrix.test-mode }}
+ changed-catalogs-contrib-modules: ${{
needs.changes.outputs.changed_catalogs_contrib_modules }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ad99fad326..fae1b875c0 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -5,8 +5,18 @@ on:
# Triggers the workflow on push or pull request events but only for the
"main" branch
push:
branches: [ "main", "branch-*" ]
+ paths-ignore:
+ - '**/*.md'
+ - '**/*.mdx'
+ - 'docs/assets/**'
+ - 'web-v2/**'
pull_request:
branches: [ "main", "branch-*" ]
+ paths-ignore:
+ - '**/*.md'
+ - '**/*.mdx'
+ - 'docs/assets/**'
+ - 'web-v2/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.ref }}
@@ -42,8 +52,13 @@ jobs:
- '**/resources/**'
- '**/src/**'
- '!**/*.md'
- - '!**/*.png'
- - '!**/*.svg'
+ - '!**/*.mdx'
+ - '!docs/**/*.png'
+ - '!docs/**/*.svg'
+ - '!docs/**/*.jpg'
+ - '!docs/**/*.jpeg'
+ - '!docs/**/*.gif'
+ - '!docs/**/*.webp'
- 'mcp-server/**'
spark_connector_changes:
- spark-connector/**
@@ -143,7 +158,7 @@ jobs:
- name: Build with Gradle (skip mcp-server tests)
if: needs.changes.outputs.mcp_server_changes != 'true'
- run: ./gradlew build -PskipITs -PskipDockerTests=false -x
:clients:client-python:build -x :mcp-server:build -x :mcp-server:test -x
:mcp-server:pylint
+ run: ./gradlew build -PskipITs -PskipDockerTests=false -x
:clients:client-python:build -x :mcp-server:build -x :mcp-server:test -x
:mcp-server:pylint -x :web-v2:web:build
- name: Fetch base branch for coverage diff
if: github.event_name == 'pull_request'
diff --git a/.github/workflows/cron-integration-test.yml
b/.github/workflows/cron-integration-test.yml
index 8f38b17b31..7acdcb3f32 100644
--- a/.github/workflows/cron-integration-test.yml
+++ b/.github/workflows/cron-integration-test.yml
@@ -42,7 +42,7 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
diff --git a/.github/workflows/flink-integration-test.yml
b/.github/workflows/flink-integration-test.yml
index ee4749c292..dd1696afd2 100644
--- a/.github/workflows/flink-integration-test.yml
+++ b/.github/workflows/flink-integration-test.yml
@@ -43,7 +43,7 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
diff --git a/.github/workflows/frontend-integration-test.yml
b/.github/workflows/frontend-integration-test.yml
index ec8b03c58c..e6bd894fa8 100644
--- a/.github/workflows/frontend-integration-test.yml
+++ b/.github/workflows/frontend-integration-test.yml
@@ -24,24 +24,32 @@ jobs:
- .github/**
- api/**
- bin/**
+ - authorizations/**
+ - bundles/**
- catalogs/**
- - clients/client-java/**
+ - catalogs-contrib/**
+ - clients/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- iceberg/**
- - integration-test-common/**
- - meta/**
+ - lance/**
- scripts/**
- - server/**
+ - server/**
+ - meta/**
- server-common/**
+ - integration-test-common/**
- web-v2/**
+ - '**/*.kts'
+ - '**/*.py'
+ - '**/*.properties'
+ - '**/*.sh'
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
@@ -84,7 +92,6 @@ jobs:
- name: Frontend Integration Test
id: integrationTest
run: |
- GRAVITINO_USE_WEB_V2=true ./gradlew -PskipTests -PtestMode=embedded
-PskipDockerTests=false :web-v2:integration-test:test
GRAVITINO_USE_WEB_V2=true ./gradlew -PskipTests -PtestMode=deploy
-PskipDockerTests=false :web-v2:integration-test:test
- name: Upload integrate tests reports
diff --git a/.github/workflows/gvfs-fuse-build-test.yml
b/.github/workflows/gvfs-fuse-build-test.yml
index a2f0fb8040..c87fe8e2d0 100644
--- a/.github/workflows/gvfs-fuse-build-test.yml
+++ b/.github/workflows/gvfs-fuse-build-test.yml
@@ -40,7 +40,7 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
diff --git a/.github/workflows/python-integration-test.yml
b/.github/workflows/python-integration-test.yml
index e47fd590f1..3ba95259b4 100644
--- a/.github/workflows/python-integration-test.yml
+++ b/.github/workflows/python-integration-test.yml
@@ -40,7 +40,7 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
diff --git a/.github/workflows/spark-integration-test.yml
b/.github/workflows/spark-integration-test.yml
index 0d1b53634f..033c6aa1fc 100644
--- a/.github/workflows/spark-integration-test.yml
+++ b/.github/workflows/spark-integration-test.yml
@@ -42,7 +42,7 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
diff --git a/.github/workflows/trino-integration-test.yml
b/.github/workflows/trino-integration-test.yml
index ae9c0d5ec4..c60e91713e 100644
--- a/.github/workflows/trino-integration-test.yml
+++ b/.github/workflows/trino-integration-test.yml
@@ -42,7 +42,7 @@ jobs:
- build.gradle.kts
- gradle.properties
- gradlew
- - setting.gradle.kts
+ - settings.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
