This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 068e97db84b HBASE-29838 Run Hadoop Check as a GitHub Action
068e97db84b is described below
commit 068e97db84b96c2e9a19db2ec77bb7d88bf61534
Author: Nick Dimiduk <[email protected]>
AuthorDate: Fri Feb 6 16:13:25 2026 +0100
HBASE-29838 Run Hadoop Check as a GitHub Action
* HBASE-29838 Run Hadoop Check as a GitHub Action
Signed-off-by: Duo Zhang <[email protected]>
* adjustments for branch-2
---------
Signed-off-by: Duo Zhang <[email protected]>
---
.github/workflows/yetus-general-check.yml | 2 +-
...k.yml => yetus-jdk11-hadoop3-compile-check.yml} | 28 ++++---
...heck.yml => yetus-jdk11-hadoop3-unit-check.yml} | 43 ++++++++---
...k.yml => yetus-jdk17-hadoop3-compile-check.yml} | 34 ++++-----
...heck.yml => yetus-jdk17-hadoop3-unit-check.yml} | 49 +++++++++----
...ck.yml => yetus-jdk8-hadoop2-compile-check.yml} | 35 ++++-----
...check.yml => yetus-jdk8-hadoop2-unit-check.yml} | 50 +++++++++----
dev-support/hbase-personality.sh | 3 +
dev-support/jenkins_precommit_github_yetus.sh | 4 +
dev-support/yetus_console_to_md.py | 80 +++++++++++++++++++-
pom.xml | 85 ++++++++++++++++++++++
11 files changed, 318 insertions(+), 95 deletions(-)
diff --git a/.github/workflows/yetus-general-check.yml
b/.github/workflows/yetus-general-check.yml
index e160684ce4c..54d68d67a33 100644
--- a/.github/workflows/yetus-general-check.yml
+++ b/.github/workflows/yetus-general-check.yml
@@ -81,7 +81,7 @@ jobs:
PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
SET_JAVA_HOME: "/usr/lib/jvm/java-11"
SOURCEDIR: "${{ github.workspace }}/src"
- TESTS_FILTER:
"checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ TESTS_FILTER:
"checkstyle,javac,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
diff --git a/.github/workflows/yetus-general-check.yml
b/.github/workflows/yetus-jdk11-hadoop3-compile-check.yml
similarity index 79%
copy from .github/workflows/yetus-general-check.yml
copy to .github/workflows/yetus-jdk11-hadoop3-compile-check.yml
index e160684ce4c..3b8b2a22524 100644
--- a/.github/workflows/yetus-general-check.yml
+++ b/.github/workflows/yetus-jdk11-hadoop3-compile-check.yml
@@ -17,7 +17,7 @@
# yamllint disable rule:line-length
---
-name: Yetus General Check
+name: Yetus JDK11 Hadoop3 Compile Check
"on":
pull_request:
@@ -28,9 +28,9 @@ permissions:
statuses: write
jobs:
- general-check:
+ jdk11-hadoop3-compile-check:
runs-on: ubuntu-latest
- timeout-minutes: 600
+ timeout-minutes: 60
env:
YETUS_VERSION: '0.15.0'
@@ -68,29 +68,27 @@ jobs:
tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION
}}-bin.tar.gz
rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
- - name: Run Yetus General Check
+ - name: Run Yetus JDK11 Hadoop3 Compile Check
env:
ARCHIVE_PATTERN_LIST:
"TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
- CHANGE_TARGET: "${{ github.base_ref }}"
DOCKERFILE: "${{ github.workspace
}}/src/dev-support/docker/Dockerfile"
GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USER: ${{ github.actor }}
HADOOP_PROFILE: "3.0"
JAVA8_HOME: "/usr/lib/jvm/java-8"
- PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
- PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
+ PATCHDIR: "${{ github.workspace
}}/yetus-jdk11-hadoop3-compile-check/output"
+ PLUGINS:
"compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars"
SET_JAVA_HOME: "/usr/lib/jvm/java-11"
SOURCEDIR: "${{ github.workspace }}/src"
- TESTS_FILTER:
"checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ TESTS_FILTER: "javac,javadoc"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
- EXCLUDE_TESTS_URL:
"https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref
}}/lastSuccessfulBuild/artifact/output/excludes"
BUILD_THREAD: "4"
- SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
- SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
BRANCH_NAME: "${{ github.base_ref }}"
+ CHANGE_TARGET: "${{ github.base_ref }}"
+ SKIP_ERRORPRONE: 'true'
DEBUG: 'true'
run: |
cd "${{ github.workspace }}"
@@ -100,12 +98,12 @@ jobs:
if: always()
run: |
cd "${{ github.workspace }}"
- python3 src/dev-support/yetus_console_to_md.py
yetus-general-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+ python3 src/dev-support/yetus_console_to_md.py
yetus-jdk11-hadoop3-compile-check/output/console.txt >> $GITHUB_STEP_SUMMARY
- - name: Publish Test Results
+ - name: Publish Results
if: always()
uses: actions/upload-artifact@v4
with:
- name: yetus-general-check-output
- path: ${{ github.workspace }}/yetus-general-check/output
+ name: yetus-jdk11-hadoop3-compile-check-output
+ path: ${{ github.workspace
}}/yetus-jdk11-hadoop3-compile-check/output
retention-days: 7
diff --git a/.github/workflows/yetus-general-check.yml
b/.github/workflows/yetus-jdk11-hadoop3-unit-check.yml
similarity index 73%
copy from .github/workflows/yetus-general-check.yml
copy to .github/workflows/yetus-jdk11-hadoop3-unit-check.yml
index e160684ce4c..1b43813a7f0 100644
--- a/.github/workflows/yetus-general-check.yml
+++ b/.github/workflows/yetus-jdk11-hadoop3-unit-check.yml
@@ -17,7 +17,7 @@
# yamllint disable rule:line-length
---
-name: Yetus General Check
+name: Yetus JDK11 Hadoop3 Unit Check
"on":
pull_request:
@@ -28,9 +28,28 @@ permissions:
statuses: write
jobs:
- general-check:
+ jdk11-hadoop3-unit-check:
runs-on: ubuntu-latest
- timeout-minutes: 600
+ timeout-minutes: 360
+
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: "small"
+ test_profile: "runSmallTests"
+ - name: "medium"
+ test_profile: "runMediumTests"
+ # Large tests split alphabetically by class name (after "Test"
prefix)
+ # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]*
+ - name: "large-wave-1"
+ test_profile: "runLargeTests-wave1"
+ - name: "large-wave-2"
+ test_profile: "runLargeTests-wave2"
+ - name: "large-wave-3"
+ test_profile: "runLargeTests-wave3"
+
+ name: ${{ matrix.name }}
env:
YETUS_VERSION: '0.15.0'
@@ -68,20 +87,19 @@ jobs:
tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION
}}-bin.tar.gz
rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
- - name: Run Yetus General Check
+ - name: Run Yetus JDK11 Hadoop3 Unit Check
env:
ARCHIVE_PATTERN_LIST:
"TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
- CHANGE_TARGET: "${{ github.base_ref }}"
DOCKERFILE: "${{ github.workspace
}}/src/dev-support/docker/Dockerfile"
GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USER: ${{ github.actor }}
HADOOP_PROFILE: "3.0"
JAVA8_HOME: "/usr/lib/jvm/java-8"
- PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
- PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
+ PATCHDIR: "${{ github.workspace
}}/yetus-jdk11-hadoop3-unit-check/output"
+ PLUGINS: "github,htmlout,maven,unit"
SET_JAVA_HOME: "/usr/lib/jvm/java-11"
SOURCEDIR: "${{ github.workspace }}/src"
- TESTS_FILTER:
"checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ TESTS_FILTER: "mvninstall"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
@@ -91,7 +109,10 @@ jobs:
SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
BRANCH_NAME: "${{ github.base_ref }}"
+ CHANGE_TARGET: "${{ github.base_ref }}"
+ SKIP_ERRORPRONE: 'true'
DEBUG: 'true'
+ TEST_PROFILE: ${{ matrix.test_profile }}
run: |
cd "${{ github.workspace }}"
bash src/dev-support/jenkins_precommit_github_yetus.sh
@@ -100,12 +121,12 @@ jobs:
if: always()
run: |
cd "${{ github.workspace }}"
- python3 src/dev-support/yetus_console_to_md.py
yetus-general-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+ python3 src/dev-support/yetus_console_to_md.py
yetus-jdk11-hadoop3-unit-check/output/console.txt >> $GITHUB_STEP_SUMMARY
- name: Publish Test Results
if: always()
uses: actions/upload-artifact@v4
with:
- name: yetus-general-check-output
- path: ${{ github.workspace }}/yetus-general-check/output
+ name: yetus-jdk11-hadoop3-unit-check-${{ matrix.name }}
+ path: ${{ github.workspace }}/yetus-jdk11-hadoop3-unit-check/output
retention-days: 7
diff --git a/.github/workflows/yetus-general-check.yml
b/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml
similarity index 76%
copy from .github/workflows/yetus-general-check.yml
copy to .github/workflows/yetus-jdk17-hadoop3-compile-check.yml
index e160684ce4c..0f4a71110bf 100644
--- a/.github/workflows/yetus-general-check.yml
+++ b/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml
@@ -17,7 +17,7 @@
# yamllint disable rule:line-length
---
-name: Yetus General Check
+name: Yetus JDK17 Hadoop3 Compile Check
"on":
pull_request:
@@ -28,9 +28,9 @@ permissions:
statuses: write
jobs:
- general-check:
+ jdk17-hadoop3-compile-check:
runs-on: ubuntu-latest
- timeout-minutes: 600
+ timeout-minutes: 60
env:
YETUS_VERSION: '0.15.0'
@@ -42,10 +42,10 @@ jobs:
path: src
fetch-depth: 0
- - name: Set up JDK 11
+ - name: Set up JDK 17
uses: actions/setup-java@v4
with:
- java-version: '11'
+ java-version: '17'
distribution: 'temurin'
- name: Maven cache
@@ -68,29 +68,27 @@ jobs:
tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION
}}-bin.tar.gz
rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
- - name: Run Yetus General Check
+ - name: Run Yetus JDK17 Hadoop3 Compile Check
env:
ARCHIVE_PATTERN_LIST:
"TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
- CHANGE_TARGET: "${{ github.base_ref }}"
DOCKERFILE: "${{ github.workspace
}}/src/dev-support/docker/Dockerfile"
GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USER: ${{ github.actor }}
HADOOP_PROFILE: "3.0"
JAVA8_HOME: "/usr/lib/jvm/java-8"
- PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
- PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
- SET_JAVA_HOME: "/usr/lib/jvm/java-11"
+ PATCHDIR: "${{ github.workspace
}}/yetus-jdk17-hadoop3-compile-check/output"
+ PLUGINS:
"compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars"
+ SET_JAVA_HOME: "/usr/lib/jvm/java-17"
SOURCEDIR: "${{ github.workspace }}/src"
- TESTS_FILTER:
"checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ TESTS_FILTER: "javac,javadoc"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
- EXCLUDE_TESTS_URL:
"https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref
}}/lastSuccessfulBuild/artifact/output/excludes"
BUILD_THREAD: "4"
- SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
- SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
BRANCH_NAME: "${{ github.base_ref }}"
+ CHANGE_TARGET: "${{ github.base_ref }}"
+ SKIP_ERRORPRONE: 'true'
DEBUG: 'true'
run: |
cd "${{ github.workspace }}"
@@ -100,12 +98,12 @@ jobs:
if: always()
run: |
cd "${{ github.workspace }}"
- python3 src/dev-support/yetus_console_to_md.py
yetus-general-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+ python3 src/dev-support/yetus_console_to_md.py
yetus-jdk17-hadoop3-compile-check/output/console.txt >> $GITHUB_STEP_SUMMARY
- - name: Publish Test Results
+ - name: Publish Results
if: always()
uses: actions/upload-artifact@v4
with:
- name: yetus-general-check-output
- path: ${{ github.workspace }}/yetus-general-check/output
+ name: yetus-jdk17-hadoop3-compile-check-output
+ path: ${{ github.workspace
}}/yetus-jdk17-hadoop3-compile-check/output
retention-days: 7
diff --git a/.github/workflows/yetus-general-check.yml
b/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml
similarity index 71%
copy from .github/workflows/yetus-general-check.yml
copy to .github/workflows/yetus-jdk17-hadoop3-unit-check.yml
index e160684ce4c..8b39b9ee53f 100644
--- a/.github/workflows/yetus-general-check.yml
+++ b/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml
@@ -17,7 +17,7 @@
# yamllint disable rule:line-length
---
-name: Yetus General Check
+name: Yetus JDK17 Hadoop3 Unit Check
"on":
pull_request:
@@ -28,9 +28,28 @@ permissions:
statuses: write
jobs:
- general-check:
+ jdk17-hadoop3-unit-check:
runs-on: ubuntu-latest
- timeout-minutes: 600
+ timeout-minutes: 360
+
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: "small"
+ test_profile: "runSmallTests"
+ - name: "medium"
+ test_profile: "runMediumTests"
+ # Large tests split alphabetically by class name (after "Test"
prefix)
+ # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]*
+ - name: "large-wave-1"
+ test_profile: "runLargeTests-wave1"
+ - name: "large-wave-2"
+ test_profile: "runLargeTests-wave2"
+ - name: "large-wave-3"
+ test_profile: "runLargeTests-wave3"
+
+ name: ${{ matrix.name }}
env:
YETUS_VERSION: '0.15.0'
@@ -42,10 +61,10 @@ jobs:
path: src
fetch-depth: 0
- - name: Set up JDK 11
+ - name: Set up JDK 17
uses: actions/setup-java@v4
with:
- java-version: '11'
+ java-version: '17'
distribution: 'temurin'
- name: Maven cache
@@ -68,20 +87,19 @@ jobs:
tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION
}}-bin.tar.gz
rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
- - name: Run Yetus General Check
+ - name: Run Yetus JDK17 Hadoop3 Unit Check
env:
ARCHIVE_PATTERN_LIST:
"TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
- CHANGE_TARGET: "${{ github.base_ref }}"
DOCKERFILE: "${{ github.workspace
}}/src/dev-support/docker/Dockerfile"
GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USER: ${{ github.actor }}
HADOOP_PROFILE: "3.0"
JAVA8_HOME: "/usr/lib/jvm/java-8"
- PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
- PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
- SET_JAVA_HOME: "/usr/lib/jvm/java-11"
+ PATCHDIR: "${{ github.workspace
}}/yetus-jdk17-hadoop3-unit-check/output"
+ PLUGINS: "github,htmlout,maven,unit"
+ SET_JAVA_HOME: "/usr/lib/jvm/java-17"
SOURCEDIR: "${{ github.workspace }}/src"
- TESTS_FILTER:
"checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ TESTS_FILTER: "mvninstall"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
@@ -91,7 +109,10 @@ jobs:
SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
BRANCH_NAME: "${{ github.base_ref }}"
+ CHANGE_TARGET: "${{ github.base_ref }}"
+ SKIP_ERRORPRONE: 'true'
DEBUG: 'true'
+ TEST_PROFILE: ${{ matrix.test_profile }}
run: |
cd "${{ github.workspace }}"
bash src/dev-support/jenkins_precommit_github_yetus.sh
@@ -100,12 +121,12 @@ jobs:
if: always()
run: |
cd "${{ github.workspace }}"
- python3 src/dev-support/yetus_console_to_md.py
yetus-general-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+ python3 src/dev-support/yetus_console_to_md.py
yetus-jdk17-hadoop3-unit-check/output/console.txt >> $GITHUB_STEP_SUMMARY
- name: Publish Test Results
if: always()
uses: actions/upload-artifact@v4
with:
- name: yetus-general-check-output
- path: ${{ github.workspace }}/yetus-general-check/output
+ name: yetus-jdk17-hadoop3-unit-check-${{ matrix.name }}
+ path: ${{ github.workspace }}/yetus-jdk17-hadoop3-unit-check/output
retention-days: 7
diff --git a/.github/workflows/yetus-general-check.yml
b/.github/workflows/yetus-jdk8-hadoop2-compile-check.yml
similarity index 75%
copy from .github/workflows/yetus-general-check.yml
copy to .github/workflows/yetus-jdk8-hadoop2-compile-check.yml
index e160684ce4c..60159e4c94e 100644
--- a/.github/workflows/yetus-general-check.yml
+++ b/.github/workflows/yetus-jdk8-hadoop2-compile-check.yml
@@ -17,7 +17,7 @@
# yamllint disable rule:line-length
---
-name: Yetus General Check
+name: Yetus JDK8 Hadoop2 Compile Check
"on":
pull_request:
@@ -28,9 +28,9 @@ permissions:
statuses: write
jobs:
- general-check:
+ jdk8-hadoop2-compile-check:
runs-on: ubuntu-latest
- timeout-minutes: 600
+ timeout-minutes: 60
env:
YETUS_VERSION: '0.15.0'
@@ -42,10 +42,10 @@ jobs:
path: src
fetch-depth: 0
- - name: Set up JDK 11
+ - name: Set up JDK 8
uses: actions/setup-java@v4
with:
- java-version: '11'
+ java-version: '8'
distribution: 'temurin'
- name: Maven cache
@@ -68,29 +68,26 @@ jobs:
tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION
}}-bin.tar.gz
rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
- - name: Run Yetus General Check
+ - name: Run Yetus JDK8 Hadoop2 Compile Check
env:
ARCHIVE_PATTERN_LIST:
"TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
- CHANGE_TARGET: "${{ github.base_ref }}"
DOCKERFILE: "${{ github.workspace
}}/src/dev-support/docker/Dockerfile"
GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USER: ${{ github.actor }}
- HADOOP_PROFILE: "3.0"
JAVA8_HOME: "/usr/lib/jvm/java-8"
- PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
- PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
- SET_JAVA_HOME: "/usr/lib/jvm/java-11"
+ PATCHDIR: "${{ github.workspace
}}/yetus-jdk8-hadoop2-compile-check/output"
+ PLUGINS:
"compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars"
+ SET_JAVA_HOME: "/usr/lib/jvm/java-8"
SOURCEDIR: "${{ github.workspace }}/src"
- TESTS_FILTER:
"checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ TESTS_FILTER: "javac,javadoc"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
- EXCLUDE_TESTS_URL:
"https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref
}}/lastSuccessfulBuild/artifact/output/excludes"
BUILD_THREAD: "4"
- SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
- SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
BRANCH_NAME: "${{ github.base_ref }}"
+ CHANGE_TARGET: "${{ github.base_ref }}"
+ SKIP_ERRORPRONE: 'true'
DEBUG: 'true'
run: |
cd "${{ github.workspace }}"
@@ -100,12 +97,12 @@ jobs:
if: always()
run: |
cd "${{ github.workspace }}"
- python3 src/dev-support/yetus_console_to_md.py
yetus-general-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+ python3 src/dev-support/yetus_console_to_md.py
yetus-jdk8-hadoop2-compile-check/output/console.txt >> $GITHUB_STEP_SUMMARY
- - name: Publish Test Results
+ - name: Publish Results
if: always()
uses: actions/upload-artifact@v4
with:
- name: yetus-general-check-output
- path: ${{ github.workspace }}/yetus-general-check/output
+ name: yetus-jdk8-hadoop2-compile-check-output
+ path: ${{ github.workspace }}/yetus-jdk8-hadoop2-compile-check/output
retention-days: 7
diff --git a/.github/workflows/yetus-general-check.yml
b/.github/workflows/yetus-jdk8-hadoop2-unit-check.yml
similarity index 71%
copy from .github/workflows/yetus-general-check.yml
copy to .github/workflows/yetus-jdk8-hadoop2-unit-check.yml
index e160684ce4c..283bccda071 100644
--- a/.github/workflows/yetus-general-check.yml
+++ b/.github/workflows/yetus-jdk8-hadoop2-unit-check.yml
@@ -17,7 +17,7 @@
# yamllint disable rule:line-length
---
-name: Yetus General Check
+name: Yetus JDK8 Hadoop2 Unit Check
"on":
pull_request:
@@ -28,9 +28,28 @@ permissions:
statuses: write
jobs:
- general-check:
+ jdk8-hadoop2-unit-check:
runs-on: ubuntu-latest
- timeout-minutes: 600
+ timeout-minutes: 360
+
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: "small"
+ test_profile: "runSmallTests"
+ - name: "medium"
+ test_profile: "runMediumTests"
+ # Large tests split alphabetically by class name (after "Test"
prefix)
+ # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]*
+ - name: "large-wave-1"
+ test_profile: "runLargeTests-wave1"
+ - name: "large-wave-2"
+ test_profile: "runLargeTests-wave2"
+ - name: "large-wave-3"
+ test_profile: "runLargeTests-wave3"
+
+ name: ${{ matrix.name }}
env:
YETUS_VERSION: '0.15.0'
@@ -42,10 +61,10 @@ jobs:
path: src
fetch-depth: 0
- - name: Set up JDK 11
+ - name: Set up JDK 8
uses: actions/setup-java@v4
with:
- java-version: '11'
+ java-version: '8'
distribution: 'temurin'
- name: Maven cache
@@ -68,20 +87,18 @@ jobs:
tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION
}}-bin.tar.gz
rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
- - name: Run Yetus General Check
+ - name: Run Yetus JDK8 Hadoop2 Unit Check
env:
ARCHIVE_PATTERN_LIST:
"TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
- CHANGE_TARGET: "${{ github.base_ref }}"
DOCKERFILE: "${{ github.workspace
}}/src/dev-support/docker/Dockerfile"
GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USER: ${{ github.actor }}
- HADOOP_PROFILE: "3.0"
JAVA8_HOME: "/usr/lib/jvm/java-8"
- PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
- PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
- SET_JAVA_HOME: "/usr/lib/jvm/java-11"
+ PATCHDIR: "${{ github.workspace
}}/yetus-jdk8-hadoop2-unit-check/output"
+ PLUGINS: "github,htmlout,maven,unit"
+ SET_JAVA_HOME: "/usr/lib/jvm/java-8"
SOURCEDIR: "${{ github.workspace }}/src"
- TESTS_FILTER:
"checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ TESTS_FILTER: "mvninstall"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
@@ -91,7 +108,10 @@ jobs:
SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
BRANCH_NAME: "${{ github.base_ref }}"
+ CHANGE_TARGET: "${{ github.base_ref }}"
+ SKIP_ERRORPRONE: 'true'
DEBUG: 'true'
+ TEST_PROFILE: ${{ matrix.test_profile }}
run: |
cd "${{ github.workspace }}"
bash src/dev-support/jenkins_precommit_github_yetus.sh
@@ -100,12 +120,12 @@ jobs:
if: always()
run: |
cd "${{ github.workspace }}"
- python3 src/dev-support/yetus_console_to_md.py
yetus-general-check/output/console.txt >> $GITHUB_STEP_SUMMARY
+ python3 src/dev-support/yetus_console_to_md.py
yetus-jdk8-hadoop2-unit-check/output/console.txt >> $GITHUB_STEP_SUMMARY
- name: Publish Test Results
if: always()
uses: actions/upload-artifact@v4
with:
- name: yetus-general-check-output
- path: ${{ github.workspace }}/yetus-general-check/output
+ name: yetus-jdk8-hadoop2-unit-check-${{ matrix.name }}
+ path: ${{ github.workspace }}/yetus-jdk8-hadoop2-unit-check/output
retention-days: 7
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index d20041b3e07..2f56d1bcdf6 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -330,6 +330,9 @@ function personality_file_tests
## @audience private
## @stability evolving
## @param name of variable to set with maven arguments
+# NOTE: INCLUDE_TESTS_URL uses -Dtest= which conflicts with pom.xml <include>
patterns.
+# Do not use INCLUDE_TESTS_URL with profiles that define their own
<include> patterns
+# (e.g., runLargeTests-wave1, runLargeTests-wave2, runLargeTests-wave3).
function get_include_exclude_tests_arg
{
local __resultvar=$1
diff --git a/dev-support/jenkins_precommit_github_yetus.sh
b/dev-support/jenkins_precommit_github_yetus.sh
index d5d61f69cfc..f9025fb110f 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -167,6 +167,10 @@ fi
if [[ -n "${JAVA8_HOME}" ]]; then
YETUS_ARGS+=("--java8-home=${JAVA8_HOME}")
fi
+# Test profile for running specific test categories (e.g., runDevTests,
runLargeTests-wave1)
+if [[ -n "${TEST_PROFILE}" ]]; then
+ YETUS_ARGS+=("--test-profile=${TEST_PROFILE}")
+fi
echo "Launching yetus with command line:"
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
diff --git a/dev-support/yetus_console_to_md.py
b/dev-support/yetus_console_to_md.py
index 3f32995e545..c85806b3cf7 100644
--- a/dev-support/yetus_console_to_md.py
+++ b/dev-support/yetus_console_to_md.py
@@ -22,7 +22,7 @@ Convert Apache Yetus console output to Markdown format.
import re
import sys
from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Tuple
# Vote to emoji mapping
@@ -170,6 +170,72 @@ def process_first_table(lines: List[str], start_idx: int)
-> Tuple[List[str], in
return content, i
+# TODO: Yetus should support this natively, but docker integration with job
summaries doesn't seem
+# to work out of the box.
+def extract_failed_tests_from_unit_files(output_dir: Path) -> List[Tuple[str,
List[str]]]:
+ """
+ Extract failed test names from patch-unit-*.txt files.
+
+ Parses Maven surefire output to find lines like:
+ [ERROR] org.apache.hadoop.hbase.types.TestPBCell.testRoundTrip
+
+ Returns:
+ List of (module_name, [failed_test_names]) tuples
+ """
+ results = []
+
+ for unit_file in output_dir.glob('patch-unit-*.txt'):
+ module_name = unit_file.stem.replace('patch-unit-', '')
+ failed_tests = set()
+
+ with open(unit_file, 'r') as f:
+ in_failures_section = False
+ for line in f:
+ stripped = line.strip()
+
+ if stripped == '[ERROR] Failures:':
+ in_failures_section = True
+ continue
+
+ if in_failures_section:
+ if stripped.startswith('[ERROR]') and not
stripped.startswith('[ERROR] Run'):
+ test_name = stripped.replace('[ERROR] ', '').strip()
+ if test_name and '.' in test_name:
+ failed_tests.add(test_name)
+ elif stripped.startswith('[INFO]') or not stripped:
+ in_failures_section = False
+
+ if failed_tests:
+ results.append((module_name, sorted(failed_tests)))
+
+ return results
+
+
+def format_failed_tests_section(failed_tests: List[Tuple[str, List[str]]]) ->
List[str]:
+ """
+ Format failed tests into markdown.
+
+ Args:
+ failed_tests: List of (module_name, [test_names]) tuples
+
+ Returns:
+ List of markdown lines
+ """
+ if not failed_tests:
+ return []
+
+ content = []
+ content.append('\n## ❌ Failed Tests\n\n')
+ content.append('| Module | Failed Tests |\n')
+ content.append('|--------|-------------|\n')
+
+ for module_name, tests in failed_tests:
+ tests_str = ', '.join(tests)
+ content.append(f'| {module_name} | {tests_str} |\n')
+
+ return content
+
+
def process_second_table(lines: List[str], start_idx: int) -> Tuple[List[str],
int]:
"""
Process the second table (Subsystem, Report/Notes).
@@ -206,13 +272,17 @@ def process_second_table(lines: List[str], start_idx:
int) -> Tuple[List[str], i
return content, i
-def convert_console_to_markdown(input_file: str, output_file: str | None =
None) -> str:
+def convert_console_to_markdown(input_file: str, output_file: Optional[str] =
None) -> str:
"""Convert console to Markdown format."""
+ input_path = Path(input_file)
+ output_dir = input_path.parent
+
with open(input_file, 'r') as f:
lines = f.readlines()
content = []
i = 0
+ added_failed_tests = False
while i < len(lines):
line = lines[i]
@@ -233,6 +303,12 @@ def convert_console_to_markdown(input_file: str,
output_file: str | None = None)
if '| Vote |' in line and 'Subsystem' in line:
table_content, i = process_first_table(lines, i + 1)
content.extend(table_content)
+
+ # Extract and add failed tests from patch-unit-*.txt files
+ if not added_failed_tests:
+ failed_tests = extract_failed_tests_from_unit_files(output_dir)
+ content.extend(format_failed_tests_section(failed_tests))
+ added_failed_tests = True
continue
# Detect second table start
diff --git a/pom.xml b/pom.xml
index 26a3fb5aeac..4c6b81d9b1c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4703,6 +4703,91 @@
<surefire.secondPartGroups/>
</properties>
</profile>
+ <profile>
+ <!--
+ Wave profiles for partitioning Large tests across parallel CI jobs.
+ Tests are split alphabetically by class name (after "Test" prefix):
+ Wave 1: Test[A-H]* classes
+ Wave 2: Test[I-R]* classes
+ Wave 3: Test[S-Z]* classes
+
+ Flaky test exclusion is handled separately via -Dtest.exclude.pattern
+ which is set by hbase-personality.sh from EXCLUDE_TESTS_URL.
+ -->
+ <id>runLargeTests-wave1</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <surefire.skipFirstPart>false</surefire.skipFirstPart>
+ <surefire.skipSecondPart>true</surefire.skipSecondPart>
+
<surefire.firstPartGroups>org.apache.hadoop.hbase.testclassification.LargeTests</surefire.firstPartGroups>
+ <surefire.secondPartGroups/>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <includes>
+ <include>%regex[(?i).*/test[a-h].*]</include>
+ </includes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>runLargeTests-wave2</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <surefire.skipFirstPart>false</surefire.skipFirstPart>
+ <surefire.skipSecondPart>true</surefire.skipSecondPart>
+
<surefire.firstPartGroups>org.apache.hadoop.hbase.testclassification.LargeTests</surefire.firstPartGroups>
+ <surefire.secondPartGroups/>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <includes>
+ <include>%regex[(?i).*/test[i-r].*]</include>
+ </includes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>runLargeTests-wave3</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <surefire.skipFirstPart>false</surefire.skipFirstPart>
+ <surefire.skipSecondPart>true</surefire.skipSecondPart>
+
<surefire.firstPartGroups>org.apache.hadoop.hbase.testclassification.LargeTests</surefire.firstPartGroups>
+ <surefire.secondPartGroups/>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <includes>
+ <include>%regex[(?i).*/test[s-z].*]</include>
+ </includes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
<profile>
<!-- Use it to launch small & medium tests -->
<id>runDevTests</id>
