This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ccce9187dcb HDDS-15749. Run specific JUnit tests if possible (#10671)
ccce9187dcb is described below
commit ccce9187dcb55a2aabd067459c73ad99ebe72923
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Oct 6 08:24:53 2026 +0200
HDDS-15749. Run specific JUnit tests if possible (#10671)
---
.github/workflows/ci.yml | 26 +++++++++++++
dev-support/ci/selective_ci_checks.bats | 42 ++++++++++++++++++--
dev-support/ci/selective_ci_checks.sh | 68 ++++++++++++++++++++++++++++++---
3 files changed, 127 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c2a2158a79e..66e32c01582 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,6 +56,7 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
+ test-classes: ${{ steps.selective-checks.outputs.test-classes }}
integration-suites: ${{ steps.integration-suites.outputs.suites }}
needs-basic-check: ${{
steps.categorize-basic-checks.outputs.needs-basic-check }}
basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
@@ -371,6 +372,31 @@ jobs:
profile: ${{ fromJson(needs.build-info.outputs.integration-suites) }}
fail-fast: false
+ # same as integration, but runs specific tests instead of profiles
+ specific-tests:
+ name: integration
+ needs:
+ - build-info
+ - build
+ - basic
+ - dependency
+ - license
+ if: |
+ needs.build-info.outputs.test-classes != '' &&
+ needs.build-info.outputs.needs-integration-tests != 'true'
+ uses: $/.github/workflows/check.yml
+ secrets:
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+ with:
+ java-version: ${{ needs.build-info.outputs.java-version }}
+ pre-script: sudo hostname localhost
+ ratis-args: ${{ inputs.ratis_args }}
+ script: integration
+ script-args: -Drocks_tools_native -DskipShade -Dtest=${{
needs.build-info.outputs.test-classes }}
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 90
+ with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
+
coverage:
runs-on: ubuntu-24.04
timeout-minutes: 30
diff --git a/dev-support/ci/selective_ci_checks.bats
b/dev-support/ci/selective_ci_checks.bats
index a2579dec369..c012d4e61de 100644
--- a/dev-support/ci/selective_ci_checks.bats
+++ b/dev-support/ci/selective_ci_checks.bats
@@ -108,6 +108,7 @@ load bats-assert/load.bash
assert_output -p needs-compose-tests=false
assert_output -p needs-integration-tests=true
assert_output -p needs-kubernetes-tests=false
+ assert_output -p test-classes=
}
@test "integration and unit: java change" {
@@ -117,8 +118,9 @@ load bats-assert/load.bash
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=false
- assert_output -p needs-integration-tests=true
+ assert_output -p needs-integration-tests=false
assert_output -p needs-kubernetes-tests=false
+ assert_output -p
test-classes=org/apache/ozone/test/Test*,org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java,org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesWithFSO.java,org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java
}
@test "integration and unit: script change" {
@@ -130,6 +132,31 @@ load bats-assert/load.bash
assert_output -p needs-compose-tests=false
assert_output -p needs-integration-tests=true
assert_output -p needs-kubernetes-tests=false
+ assert_output -p test-classes=
+}
+
+@test "integration and unit: test-only change" {
+ run dev-support/ci/selective_ci_checks.sh
bf59f31a7e5fcad1acc85e1228d77c22ee7f036c
+
+ assert_output -p
'basic-checks=["rat","author","checkstyle","errorprone","findbugs","pmd"]'
+ assert_output -p needs-build=true
+ assert_output -p needs-compile=true
+ assert_output -p needs-compose-tests=false
+ assert_output -p needs-integration-tests=false
+ assert_output -p needs-kubernetes-tests=false
+ assert_output -p
test-classes=org/apache/ozone/test/Test*,org/apache/hadoop/ozone/container/TestContainerReportHandling.java,org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java,org/apache/hadoop/ozone/container/TestHelper.java
+}
+
+@test "contract test" {
+ run dev-support/ci/selective_ci_checks.sh
c0a6ffd6dda7432f57e38bcf3f5ababdd7999e26
+
+ assert_output -p
'basic-checks=["rat","author","checkstyle","errorprone","findbugs","pmd"]'
+ assert_output -p needs-build=true
+ assert_output -p needs-compile=true
+ assert_output -p needs-compose-tests=false
+ assert_output -p needs-integration-tests=false
+ assert_output -p needs-kubernetes-tests=false
+ assert_output -p
test-classes=org/apache/ozone/test/Test*,org/apache/hadoop/fs/ozone/contract/Test*
}
@test "script change including javadoc.sh" {
@@ -152,6 +179,7 @@ load bats-assert/load.bash
assert_output -p needs-compose-tests=false
assert_output -p needs-integration-tests=true
assert_output -p needs-kubernetes-tests=false
+ assert_output -p test-classes=
}
@test "unit only" {
@@ -161,8 +189,9 @@ load bats-assert/load.bash
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=false
- assert_output -p needs-integration-tests=true
+ assert_output -p needs-integration-tests=false
assert_output -p needs-kubernetes-tests=false
+ assert_output -p
test-classes=org/apache/ozone/test/Test*,org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetOwnerRequest.java,org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetOwnerRequestWithFSO.java,org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetQuotaRequest.java,org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetQuotaRequestWithFSO.java
}
@test "unit helper" {
@@ -174,6 +203,7 @@ load bats-assert/load.bash
assert_output -p needs-compose-tests=false
assert_output -p needs-integration-tests=true
assert_output -p needs-kubernetes-tests=false
+ assert_output -p test-classes=
}
@test "integration only" {
@@ -183,8 +213,9 @@ load bats-assert/load.bash
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=false
- assert_output -p needs-integration-tests=true
+ assert_output -p needs-integration-tests=false
assert_output -p needs-kubernetes-tests=false
+ assert_output -p
test-classes=org/apache/ozone/test/Test*,org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
}
@test "mini-cluster" {
@@ -196,6 +227,7 @@ load bats-assert/load.bash
assert_output -p needs-compose-tests=false
assert_output -p needs-integration-tests=true
assert_output -p needs-kubernetes-tests=false
+ assert_output -p test-classes=
}
@test "test-utils" {
@@ -207,6 +239,7 @@ load bats-assert/load.bash
assert_output -p needs-compose-tests=false
assert_output -p needs-integration-tests=true
assert_output -p needs-kubernetes-tests=false
+ assert_output -p test-classes=
}
@test "native only" {
@@ -227,8 +260,9 @@ load bats-assert/load.bash
assert_output -p needs-build=true
assert_output -p needs-compile=true
assert_output -p needs-compose-tests=false
- assert_output -p needs-integration-tests=true
+ assert_output -p needs-integration-tests=false
assert_output -p needs-kubernetes-tests=false
+ assert_output -p
test-classes=org/apache/ozone/test/Test*,org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java
}
@test "kubernetes only" {
diff --git a/dev-support/ci/selective_ci_checks.sh
b/dev-support/ci/selective_ci_checks.sh
index b45ef77179f..ae247080de9 100755
--- a/dev-support/ci/selective_ci_checks.sh
+++ b/dev-support/ci/selective_ci_checks.sh
@@ -38,7 +38,9 @@
#
declare -a pattern_array ignore_array
+declare -i match_count
matched_files=""
+CHANGED_TEST_CLASSES=""
ignore_array=(
"^[^/]*.md" # exclude root docs
)
@@ -136,7 +138,8 @@ function get_regexp_from_patterns() {
# ignore_array - array storing regexp patterns to be ignored
# Output:
# match_count - number of matched files
-# matched_files (appended) - list of matched files
+# new_matched_files - list of matched files
+# matched_files (appended) - cumulative list of all files matched so far
function filter_changed_files() {
local add_to_list="${1:-}"
@@ -146,17 +149,20 @@ function filter_changed_files() {
verbosity::store_exit_on_error_status
- match_count=$(echo "${CHANGED_FILES}" | grep -E "${match}" | grep -cEv
"${ignore}")
+ new_matched_files=$(echo "${CHANGED_FILES}" | grep -E "${match}" | grep
-Ev "${ignore}")
+ match_count=0
+ if [[ -n "${new_matched_files}" ]]; then
+ match_count=$(echo "${new_matched_files}" | wc -l)
+ fi
if [[ "${add_to_list}" == "true" ]]; then
- local additional=$(echo "${CHANGED_FILES}" | grep -E "${match}" | grep
-Ev "${ignore}")
- matched_files="${matched_files}$(echo -e "\n${additional}")"
+ matched_files="${matched_files}$(echo -e "\n${new_matched_files}")"
fi
echo
echo "${match_count} changed files matching the '${match}' pattern, but
ignoring '${ignore}':"
echo
- echo "${CHANGED_FILES}" | grep -E "${match}" | grep -Ev "${ignore}"
+ echo "${new_matched_files}"
echo
verbosity::restore_exit_on_error_status
@@ -257,6 +263,45 @@ function get_count_doc_files() {
start_end::group_end
}
+function get_changed_test_classes() {
+ start_end::group_start "List changed test classes"
+ local pattern_array=(
+ "src/test/java/.*/Test.*\.java"
+ "src/test/java/org/apache/hadoop/fs/contract"
+ "src/test/java/org/apache/hadoop/fs/ozone/contract"
+ "src/test/java/org/apache/hadoop/tools/contract"
+ )
+ filter_changed_files true
+
+ local test_classes=""
+
+ if [[ -n "${new_matched_files}" ]]; then
+ # always run grouped integration tests
+ test_classes="org/apache/ozone/test/Test*"
+
+ local f
+ for f in ${new_matched_files}; do
+ if echo "$f" | grep -q -e '/fs/contract/' -e '/fs/ozone/contract/' -e
'/tools/contract/'; then
+ # run all contract tests
+
test_classes="${test_classes},org/apache/hadoop/fs/ozone/contract/Test*"
+ else
+ test_classes="${test_classes},${f}"
+ # add WithFSO subclass (no problem if it does not exist)
+ if echo "$f" | grep -q
'hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/re\(quest\|sponse\)'
\
+ && ! echo "$f" | grep -q 'WithFSO'; then
+ local with_fso
+ with_fso=$(echo "$f" | sed 's/\.java/WithFSO.java/')
+ test_classes="${test_classes},${with_fso}"
+ fi
+ fi
+ done
+ fi
+
+ CHANGED_TEST_CLASSES=$(echo "${test_classes}" | sed -e
's@[^,]*/src/test/java/@@g')
+ readonly CHANGED_TEST_CLASSES
+ start_end::group_end
+}
+
function get_count_integration_files() {
start_end::group_start "Count integration test files"
local pattern_array=(
@@ -269,6 +314,12 @@ function get_count_integration_files() {
"src/test/java"
"src/test/resources"
)
+ local ignore_array=(
+ "src/test/java/.*/Test.*\.java"
+ "src/test/java/org/apache/hadoop/fs/contract"
+ "src/test/java/org/apache/hadoop/fs/ozone/contract"
+ "src/test/java/org/apache/hadoop/tools/contract"
+ )
filter_changed_files true
COUNT_INTEGRATION_CHANGED_FILES=${match_count}
readonly COUNT_INTEGRATION_CHANGED_FILES
@@ -557,6 +608,12 @@ function set_outputs() {
initialization::ga_output needs-compose-tests "${compose_tests_needed}"
initialization::ga_output needs-integration-tests
"${integration_tests_needed}"
initialization::ga_output needs-kubernetes-tests
"${kubernetes_tests_needed}"
+
+ if [[ "${integration_tests_needed}" == "true" ]]; then
+ initialization::ga_output test-classes ""
+ else
+ initialization::ga_output test-classes "${CHANGED_TEST_CLASSES}"
+ fi
}
check_for_full_tests_needed_label
@@ -585,6 +642,7 @@ run_all_tests_if_environment_files_changed
get_count_all_files
get_count_compose_files
get_count_doc_files
+get_changed_test_classes
get_count_integration_files
get_count_kubernetes_files
get_count_robot_files
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
