This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new e675c0465c [#12091] improvement(ci): Route contrib catalog tests to
contrib CI (#12092)
e675c0465c is described below
commit e675c0465c2be89b5738f44d090dc5eb29aa3f48
Author: roryqi <[email protected]>
AuthorDate: Mon Jul 20 19:07:07 2026 +0800
[#12091] improvement(ci): Route contrib catalog tests to contrib CI (#12092)
### What changes were proposed in this pull request?
This PR moves heavyweight contrib catalog JDBC tests out of the general
Build and Backend Integration Test workflows.
- Exclude `:catalogs-contrib:catalog-jdbc-clickhouse:test`,
`:catalogs-contrib:catalog-jdbc-hologres:test`, and
`:catalogs-contrib:catalog-jdbc-oceanbase:test` from Build.
- Add `.github/workflows/contrib-catalog-test.yml` to run contrib
catalog tests in a dedicated pipeline.
- Detect changed `catalogs-contrib` modules and run only their Gradle
test tasks. Shared CI, Gradle, `catalogs-contrib/build.gradle.kts`,
`catalog-common`, or `catalog-jdbc-common` changes run all contrib
catalog tests.
- Remove the `catalogs-contrib` module detection and routing from
Backend Integration Test. The reusable backend action now excludes all
`catalogs-contrib` test tasks, and contrib ITs run in `Contrib Catalog
Test` without `-PskipITs`.
### Why are the changes needed?
The OceanBase CI in the Build pipeline can block unrelated PRs when its
Docker service startup is flaky. ClickHouse and Hologres are similar
contrib catalog Docker-backed tests, and contrib integration tests were
also routed through Backend Integration Test, so these tests should be
isolated in the contrib catalog workflow.
Fix: #12091
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- `git diff --check`
- YAML parse for `.github/workflows/build.yml`,
`.github/workflows/backend-integration-test.yml`,
`.github/workflows/backend-integration-test-action.yml`, and
`.github/workflows/contrib-catalog-test.yml`
- `./gradlew :catalogs-contrib:catalog-jdbc-clickhouse:testClasses
:catalogs-contrib:catalog-jdbc-hologres:testClasses
:catalogs-contrib:catalog-jdbc-oceanbase:testClasses -PskipWeb=true`
---
.../workflows/backend-integration-test-action.yml | 20 +---
.github/workflows/backend-integration-test.yml | 33 ------
.github/workflows/build.yml | 3 +
.github/workflows/contrib-catalog-test.yml | 132 +++++++++++++++++++++
4 files changed, 138 insertions(+), 50 deletions(-)
diff --git a/.github/workflows/backend-integration-test-action.yml
b/.github/workflows/backend-integration-test-action.yml
index 08b82f9331..56a7a7c6a0 100644
--- a/.github/workflows/backend-integration-test-action.yml
+++ b/.github/workflows/backend-integration-test-action.yml
@@ -20,11 +20,6 @@ on:
required: true
description: 'run on embedded or deploy mode'
type: string
- changed-catalogs-contrib-modules:
- required: false
- description: 'space-separated changed catalogs-contrib module names,
or __all__'
- type: string
- default: ''
jobs:
start-runner:
@@ -66,17 +61,10 @@ jobs:
- name: Backend Integration Test (JDK${{ inputs.java-version }}-${{
inputs.test-mode }}-${{ inputs.backend }})
id: integrationTest
run: |
- ALL_CONTRIB_MODULES="$(find catalogs-contrib -mindepth 1 -maxdepth 1
-type d -name 'catalog-*' -exec basename {} \; | sort)"
- CHANGED_CONTRIB_MODULES="${{ inputs.changed-catalogs-contrib-modules
}}"
-
EXCLUDE_CONTRIB_TESTS=""
- if [ "$CHANGED_CONTRIB_MODULES" != "__all__" ]; then
- for module in $ALL_CONTRIB_MODULES; do
- if [[ " $CHANGED_CONTRIB_MODULES " != *" $module "* ]]; then
- EXCLUDE_CONTRIB_TESTS="$EXCLUDE_CONTRIB_TESTS -x
:catalogs-contrib:$module:test"
- fi
- done
- fi
+ for module in $(find catalogs-contrib -mindepth 1 -maxdepth 1 -type
d -name 'catalog-*' -exec basename {} \; | sort); do
+ EXCLUDE_CONTRIB_TESTS="$EXCLUDE_CONTRIB_TESTS -x
:catalogs-contrib:$module:test"
+ done
./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }}
-PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false -PskipWeb=true \
-x :web:web:test -x :web:integration-test:test -x :web-v2:web:test
-x :web-v2:integration-test:test -x :clients:client-python:test \
@@ -106,7 +94,5 @@ jobs:
distribution/package/logs/*.log
catalogs/**/*.log
catalogs/**/*.tar
- catalogs-contrib/**/*.log
- catalogs-contrib/**/*.tar
distribution/**/*.log
distribution/**/*.out
diff --git a/.github/workflows/backend-integration-test.yml
b/.github/workflows/backend-integration-test.yml
index 2857db4733..6319266a68 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -27,7 +27,6 @@ jobs:
- api/**
- bin/**
- catalogs/**
- - catalogs-contrib/**
- clients/client-java/**
- clients/client-java-runtime/**
- clients/filesystem-hadoop3/**
@@ -49,38 +48,8 @@ jobs:
- gradle.properties
- gradlew
- settings.gradle.kts
- - name: Detect changed catalogs-contrib modules
- id: contrib
- run: |
- if [ "${{ github.event_name }}" = "pull_request" ]; then
- BASE_SHA="${{ github.event.pull_request.base.sha }}"
- HEAD_SHA="${{ github.event.pull_request.head.sha }}"
- else
- BASE_SHA="${{ github.event.before }}"
- HEAD_SHA="${{ github.sha }}"
- fi
-
- if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" =
"0000000000000000000000000000000000000000" ]; then
- BASE_SHA="$(git rev-list --max-parents=0 "$HEAD_SHA" | tail -n 1)"
- fi
-
- MERGE_BASE_SHA="$(git merge-base "$BASE_SHA" "$HEAD_SHA" || true)"
- if [ -z "$MERGE_BASE_SHA" ]; then
- MERGE_BASE_SHA="$BASE_SHA"
- fi
-
- CHANGED_FILES="$(git diff --name-only "$MERGE_BASE_SHA" "$HEAD_SHA")"
- CHANGED_MODULES="$(echo "$CHANGED_FILES" | awk -F/
'/^catalogs-contrib\/[^/]+\// {print $2}' | sort -u | xargs)"
-
- # If shared catalogs-contrib build logic changes, run all contrib
catalog tests.
- if echo "$CHANGED_FILES" | grep -q
'^catalogs-contrib/build.gradle.kts$'; then
- CHANGED_MODULES="__all__"
- fi
-
- echo "changed_catalogs_contrib_modules=$CHANGED_MODULES" >>
"$GITHUB_OUTPUT"
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
- changed_catalogs_contrib_modules: ${{
steps.contrib.outputs.changed_catalogs_contrib_modules }}
BackendIT-on-push:
needs: changes
@@ -104,7 +73,6 @@ jobs:
java-version: ${{ matrix.java-version }}
backend: ${{ matrix.backend }}
test-mode: ${{ matrix.test-mode }}
- changed-catalogs-contrib-modules: ${{
needs.changes.outputs.changed_catalogs_contrib_modules }}
BackendIT-on-pr:
needs: changes
@@ -128,4 +96,3 @@ jobs:
java-version: ${{ matrix.java-version }}
backend: ${{ matrix.backend }}
test-mode: ${{ matrix.test-mode }}
- changed-catalogs-contrib-modules: ${{
needs.changes.outputs.changed_catalogs_contrib_modules }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3865577d29..b41a7960ef 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -194,6 +194,9 @@ jobs:
-PskipITs
-PskipDockerTests=false
-x :clients:client-python:build
+ -x :catalogs-contrib:catalog-jdbc-clickhouse:test
+ -x :catalogs-contrib:catalog-jdbc-hologres:test
+ -x :catalogs-contrib:catalog-jdbc-oceanbase:test
)
if [ "${{ needs.changes.outputs.mcp_server_changes }}" != "true" ];
then
diff --git a/.github/workflows/contrib-catalog-test.yml
b/.github/workflows/contrib-catalog-test.yml
new file mode 100644
index 0000000000..1ab459bcfe
--- /dev/null
+++ b/.github/workflows/contrib-catalog-test.yml
@@ -0,0 +1,132 @@
+name: Contrib Catalog Test
+
+on:
+ push:
+ branches: [ "main", "branch-*" ]
+ pull_request:
+ branches: [ "main", "branch-*" ]
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ changes:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d
+ id: filter
+ with:
+ filters: |
+ source_changes:
+ - .github/workflows/contrib-catalog-test.yml
+ - catalogs-contrib/**
+ - catalogs/catalog-common/**
+ - catalogs/catalog-jdbc-common/**
+ - dev/**
+ - gradle/**
+ - integration-test-common/**
+ - build.gradle.kts
+ - gradle.properties
+ - gradlew
+ - settings.gradle.kts
+ - name: Detect changed catalogs-contrib modules
+ id: contrib
+ run: |
+ if [ "${{ github.event_name }}" = "pull_request" ]; then
+ BASE_SHA="${{ github.event.pull_request.base.sha }}"
+ HEAD_SHA="${{ github.event.pull_request.head.sha }}"
+ else
+ BASE_SHA="${{ github.event.before }}"
+ HEAD_SHA="${{ github.sha }}"
+ fi
+
+ if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" =
"0000000000000000000000000000000000000000" ]; then
+ BASE_SHA="$(git rev-list --max-parents=0 "$HEAD_SHA" | tail -n 1)"
+ fi
+
+ MERGE_BASE_SHA="$(git merge-base "$BASE_SHA" "$HEAD_SHA" || true)"
+ if [ -z "$MERGE_BASE_SHA" ]; then
+ MERGE_BASE_SHA="$BASE_SHA"
+ fi
+
+ CHANGED_FILES="$(git diff --name-only "$MERGE_BASE_SHA" "$HEAD_SHA")"
+ CHANGED_MODULES="$(echo "$CHANGED_FILES" | awk -F/
'/^catalogs-contrib\/[^/]+\// {print $2}' | sort -u | xargs)"
+
+ if echo "$CHANGED_FILES" | grep -Eq
'^(\.github/workflows/contrib-catalog-test.yml|catalogs-contrib/build.gradle.kts|catalogs/catalog-common/|catalogs/catalog-jdbc-common/|dev/|gradle/|integration-test-common/|build.gradle.kts|gradle.properties|gradlew|settings.gradle.kts)';
then
+ CHANGED_MODULES="__all__"
+ fi
+
+ echo "changed_catalogs_contrib_modules=$CHANGED_MODULES" >>
"$GITHUB_OUTPUT"
+ outputs:
+ source_changes: ${{ steps.filter.outputs.source_changes }}
+ changed_catalogs_contrib_modules: ${{
steps.contrib.outputs.changed_catalogs_contrib_modules }}
+
+ contrib-catalog-test:
+ needs: changes
+ if: needs.changes.outputs.source_changes == 'true' &&
needs.changes.outputs.changed_catalogs_contrib_modules != ''
+ runs-on: ubuntu-22.04
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-java@v4
+ with:
+ java-version: 17
+ distribution: 'temurin'
+ cache: 'gradle'
+
+ - name: Set up QEMU
+ uses:
docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
+
+ - name: Check required command
+ run: |
+ dev/ci/check_commands.sh
+
+ - name: Free up disk space
+ run: |
+ dev/ci/util_free_space.sh
+
+ - name: Install dependencies
+ run: |
+ wget
https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+ sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+
+ - name: Run contrib catalog tests
+ id: contribCatalogTest
+ run: |
+ CHANGED_CONTRIB_MODULES="${{
needs.changes.outputs.changed_catalogs_contrib_modules }}"
+
+ if [ "$CHANGED_CONTRIB_MODULES" = "__all__" ]; then
+ CONTRIB_MODULES="$(find catalogs-contrib -mindepth 1 -maxdepth 1
-type d -name 'catalog-*' -exec basename {} \; | sort)"
+ else
+ CONTRIB_MODULES="$CHANGED_CONTRIB_MODULES"
+ fi
+
+ gradle_args=()
+ for module in $CONTRIB_MODULES; do
+ if [ -d "catalogs-contrib/$module" ]; then
+ gradle_args+=(":catalogs-contrib:$module:test")
+ fi
+ done
+
+ if [ "${#gradle_args[@]}" -eq 0 ]; then
+ echo "No changed contrib catalog test tasks."
+ exit 0
+ fi
+
+ ./gradlew "${gradle_args[@]}" -PskipDockerTests=false -PskipWeb=true
+
+ - name: Upload contrib catalog test reports
+ uses: actions/upload-artifact@v7
+ if: ${{ (failure() && steps.contribCatalogTest.outcome == 'failure')
|| contains(github.event.pull_request.labels.*.name, 'upload log') }}
+ with:
+ name: contrib-catalog-test-reports
+ path: |
+ build/reports
+ catalogs-contrib/**/*.log
+ catalogs-contrib/**/*.tar
