This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 014fc51334e HBASE-28679 Upgrade yetus to a newer version (#6012)
014fc51334e is described below
commit 014fc51334e7ada07c5ad3930e3e3fa1e31129fd
Author: Duo Zhang <[email protected]>
AuthorDate: Sat Jun 22 10:22:56 2024 +0800
HBASE-28679 Upgrade yetus to a newer version (#6012)
Signed-off-by: Nick Dimiduk <[email protected]>
(cherry picked from commit 17ce7c3b4c5e58af1f70d17e62940c9dd6fb4660)
---
dev-support/Jenkinsfile | 8 ++--
dev-support/Jenkinsfile_GitHub | 55 +++++++++++++--------------
dev-support/docker/Dockerfile | 2 +
dev-support/hbase_nightly_yetus.sh | 10 ++---
dev-support/jenkins_precommit_github_yetus.sh | 13 ++++---
5 files changed, 45 insertions(+), 43 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 461df28c8e9..227229bf8d8 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -31,7 +31,7 @@ pipeline {
disableConcurrentBuilds()
}
environment {
- YETUS_RELEASE = '0.12.0'
+ YETUS_RELEASE = '0.15.0'
// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
@@ -43,12 +43,12 @@ pipeline {
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
- AUTHOR_IGNORE_LIST =
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
- WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+ AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+ BLANKS_IGNORE_LIST = '.*/generated/.*'
// output from surefire; sadly the archive function in yetus only works on
file names.
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0,
remove from here so that new problems will cause a failure.
- TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+ TESTS_FILTER =
'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
EXCLUDE_TESTS_URL =
"${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes"
// TODO does hadoopcheck need to be jdk specific?
SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general
check'
diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
index 62b16287e1e..2ea1ef697b4 100644
--- a/dev-support/Jenkinsfile_GitHub
+++ b/dev-support/Jenkinsfile_GitHub
@@ -36,14 +36,16 @@ pipeline {
YETUS_REL = 'yetus'
DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
YETUS_DRIVER_REL =
"${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
- // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
- YETUS_VERSION = 'rel/0.12.0'
+ YETUS_VERSION = '0.15.0'
GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
JDK_SPECIFIC_PLUGINS =
'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
+ // This section of the docs tells folks not to use the javadoc tag.
older branches have our old version of the check for said tag.
+ AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+ BLANKS_IGNORE_LIST = '.*/generated/.*'
// output from surefire; sadly the archive function in yetus only
works on file names.
ARCHIVE_PATTERN_LIST =
'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to
0, remove from here so that new problems will cause a failure.
- TESTS_FILTER =
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+ TESTS_FILTER =
'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
EXCLUDE_TESTS_URL =
"${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
// set build parallel
BUILD_THREAD = 4
@@ -102,11 +104,11 @@ pipeline {
checkout scm
}
dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url:
'https://github.com/apache/yetus.git']]]
- )
+ sh'''#!/usr/bin/env bash
+ wget
https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz
&& \
+ tar --strip-components=1 -xzf
apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
}
dir("${WORKDIR}") {
withCredentials([
@@ -229,11 +231,11 @@ pipeline {
checkout scm
}
dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url:
'https://github.com/apache/yetus.git']]]
- )
+ sh'''#!/usr/bin/env bash
+ wget
https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz
&& \
+ tar --strip-components=1 -xzf
apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
}
dir("${WORKDIR}") {
withCredentials([
@@ -365,11 +367,11 @@ pipeline {
checkout scm
}
dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url:
'https://github.com/apache/yetus.git']]]
- )
+ sh'''#!/usr/bin/env bash
+ wget
https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz
&& \
+ tar --strip-components=1 -xzf
apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
}
dir("${WORKDIR}") {
withCredentials([
@@ -493,22 +495,19 @@ pipeline {
SKIP_ERRORPRONE = true
}
when {
- allOf {
- // this will return true if the pipeline is
building a change request, such as a GitHub pull request.
- changeRequest()
- expression { env.CHANGE_TARGET in ['master',
'branch-3'] }
- }
+ // this will return true if the pipeline is building a
change request, such as a GitHub pull request.
+ changeRequest()
}
steps {
dir("${SOURCEDIR}") {
checkout scm
}
dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url:
'https://github.com/apache/yetus.git']]]
- )
+ sh'''#!/usr/bin/env bash
+ wget
https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz
&& \
+ tar --strip-components=1 -xzf
apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
}
dir("${WORKDIR}") {
withCredentials([
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index dcd84c89c21..499397b6313 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -53,6 +53,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
ruby=1:3.0* \
ruby-dev=1:3.0* \
shellcheck='0.8.0-*' \
+ libxml2-dev='2.9.13+dfsg-*' \
+ libxml2-utils='2.9.13+dfsg-*' \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* \
diff --git a/dev-support/hbase_nightly_yetus.sh
b/dev-support/hbase_nightly_yetus.sh
index 4c671dcfef6..3423a7e73c5 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -20,7 +20,7 @@ declare -i missing_env=0
# Validate params
for required_env in "TESTS" "PERSONALITY_FILE" "BASEDIR"
"ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE" \
"OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST" \
- "WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER"
"DEBUG" \
+ "BLANKS_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
"USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do
if [ -z "${!required_env}" ]; then
echo "[ERROR] Required environment variable '${required_env}' is not set."
@@ -59,8 +59,8 @@ YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-eol-ignore-list=${WHITESPACE_IGNORE_LIST}"
"${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-tabs-ignore-list=${WHITESPACE_IGNORE_LIST}"
"${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}"
"${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}"
"${YETUS_ARGS[@]}")
YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
@@ -106,11 +106,9 @@ if [[ -n "${JAVA8_HOME}" ]]; then
fi
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
- YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs"
"${YETUS_ARGS[@]}")
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
else
- YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-git/shelldocs/shelldocs.py"
"${YETUS_ARGS[@]}")
- TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/test-patch.sh"
+ TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/src/main/shell/test-patch.sh"
fi
echo "Launching yetus with command line:"
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
diff --git a/dev-support/jenkins_precommit_github_yetus.sh
b/dev-support/jenkins_precommit_github_yetus.sh
index 0f5de550f8e..845d4ab4e24 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -41,6 +41,8 @@ declare -a required_envs=(
"SOURCEDIR"
"TESTS_FILTER"
"YETUSDIR"
+ "AUTHOR_IGNORE_LIST"
+ "BLANKS_IGNORE_LIST"
)
# Validate params
for required_env in "${required_envs[@]}"; do
@@ -57,7 +59,7 @@ if [ ${missing_env} -gt 0 ]; then
fi
# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
-TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+TESTPATCHBIN="${YETUSDIR}/bin/test-patch"
# this must be clean for every run
rm -rf "${PATCHDIR}"
@@ -87,8 +89,8 @@ YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
# enable writing back to Github
-YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
-YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}")
+YETUS_ARGS+=("--github-write-comment")
# auto-kill any surefire stragglers during unit test runs
YETUS_ARGS+=("--reapermode=kill")
# set relatively high limits for ASF machines
@@ -109,8 +111,9 @@ YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
YETUS_ARGS+=("--mvn-custom-repos")
YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
-YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
-YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
+YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}*")
YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
YETUS_ARGS+=("--quick-hadoopcheck")
