This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 8fd1ecab8e5 HBASE-28677 Add jdk 17 task for pre commit build for 2.x
(#6002)
8fd1ecab8e5 is described below
commit 8fd1ecab8e56e7605667ab56d38a4296ee19151e
Author: Duo Zhang <[email protected]>
AuthorDate: Wed Jun 26 22:30:37 2024 +0800
HBASE-28677 Add jdk 17 task for pre commit build for 2.x (#6002)
Signed-off-by: Nick Dimiduk <[email protected]>
---
dev-support/Jenkinsfile_GitHub | 138 +++++++++++++++++++++++++++++++++++++++++
dev-support/docker/Dockerfile | 15 +++++
pom.xml | 2 +-
3 files changed, 154 insertions(+), 1 deletion(-)
diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
index b9551196ee1..c61237dbb62 100644
--- a/dev-support/Jenkinsfile_GitHub
+++ b/dev-support/Jenkinsfile_GitHub
@@ -58,6 +58,7 @@ pipeline {
WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
WORKDIR_REL_JDK8_HADOOP2_CHECK = 'yetus-jdk8-hadoop2-check'
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
+ WORKDIR_REL_JDK17_HADOOP3_CHECK = 'yetus-jdk17-hadoop3-check'
ASF_NIGHTLIES = 'https://nightlies.apache.org'
ASF_NIGHTLIES_BASE_ORI =
"${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
@@ -474,6 +475,143 @@ pipeline {
}
}
}
+ stage ('yetus jdk17 hadoop3 checks') {
+ agent {
+ node {
+ label 'hbase'
+ }
+ }
+ environment {
+ // customized per parallel stage
+ PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-17'
+ HADOOP_PROFILE = '3.0'
+ WORKDIR_REL = "${WORKDIR_REL_JDK17_HADOOP3_CHECK}"
+ // identical for all parallel stages
+ WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
+ YETUSDIR = "${WORKDIR}/${YETUS_REL}"
+ SOURCEDIR = "${WORKDIR}/${SRC_REL}"
+ PATCHDIR = "${WORKDIR}/${PATCH_REL}"
+ BUILD_URL_ARTIFACTS =
"artifact/${WORKDIR_REL}/${PATCH_REL}"
+ DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
+ YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+ SKIP_ERRORPRONE = true
+ }
+ when {
+ // this will return true if the pipeline is building a
change request, such as a GitHub pull request.
+ changeRequest()
+ }
+ steps {
+ dir("${SOURCEDIR}") {
+ checkout scm
+ }
+ dir("${YETUSDIR}") {
+ sh'''#!/usr/bin/env bash
+ wget
https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz
&& \
+ tar --strip-components=1 -xzf
apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
+ }
+ dir("${WORKDIR}") {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'apache-hbase-at-github.com',
+ passwordVariable: 'GITHUB_PASSWORD',
+ usernameVariable: 'GITHUB_USER'
+ )]) {
+ script {
+ def ret = sh(
+ label: 'test-patch',
+ returnStatus: true,
+ script: '''#!/bin/bash -e
+ hostname -a ; pwd ; ls -la
+ printenv 2>&1 | sort
+ echo "[INFO] Launching Yetus via
${YETUS_DRIVER}"
+ "${YETUS_DRIVER}"
+ '''
+ )
+ if (ret != 0) {
+ // mark the build as UNSTABLE instead of
FAILURE, to avoid skipping the later publish of
+ // test output. See HBASE-26339 for more
details.
+ currentBuild.result = 'UNSTABLE'
+ }
+ }
+ }
+ }
+ }
+ post {
+ always {
+ junit testResults:
"${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
+ allowEmptyResults: true, skipPublishingChecks:
true
+ sh label: 'zip surefire reports', script:
'''#!/bin/bash -e
+ if [ -d "${PATCHDIR}/archiver" ]; then
+ count=$(find "${PATCHDIR}/archiver" -type f
| wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${PATCHDIR}/test_logs.zip"
"${PATCHDIR}/archiver"
+ else
+ echo "No archived files, skipping
compressing."
+ fi
+ else
+ echo "No archiver directory, skipping
compressing."
+ fi
+ '''
+ sshPublisher(publishers: [
+ sshPublisherDesc(configName: 'Nightlies',
+ transfers: [
+ sshTransfer(remoteDirectory:
"hbase/${JOB_NAME}/${BUILD_NUMBER}",
+ sourceFiles:
"${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
+ )
+ ]
+ )
+ ])
+ // remove the big test logs zip file, store the
nightlies url in test_logs.txt
+ sh '''#!/bin/bash -e
+ if [ -f "${PATCHDIR}/test_logs.zip" ]; then
+ echo "Remove ${PATCHDIR}/test_logs.zip for
saving space"
+ rm -rf "${PATCHDIR}/test_logs.zip"
+ python3
${SOURCEDIR}/dev-support/gen_redirect_html.py
"${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" >
"${PATCHDIR}/test_logs.html"
+ else
+ echo "No test_logs.zip, skipping"
+ fi
+ '''
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts:
"${WORKDIR_REL}/${PATCH_REL}/*", excludes:
"${WORKDIR_REL}/${PATCH_REL}/precommit"
+ archiveArtifacts artifacts:
"${WORKDIR_REL}/${PATCH_REL}/**/*", excludes:
"${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "${WORKDIR_REL}/${PATCH_REL}",
+ reportFiles: 'report.html',
+ reportName: 'PR JDK17 Hadoop3 Check Report'
+ ]
+ }
+ // Jenkins pipeline jobs fill slaves on PRs without
this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script:
'''#!/bin/bash -e
+ # See YETUS-764
+ if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+ echo "test-patch process appears to
still be running: killing"
+ kill `cat "${PATCHDIR}/pidfile.txt"` ||
true
+ sleep 10
+ fi
+ if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+ echo "test-patch container appears to
still be running: killing"
+ docker kill `cat
"${PATCHDIR}/cidfile.txt"` || true
+ fi
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ dir ("${WORKDIR}") {
+ deleteDir()
+ }
+ }
+ }
+ }
+ }
}
}
}
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index bc8dee088ae..499397b6313 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -112,6 +112,13 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output
/tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \
echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c -
+FROM base_image AS openjdk17_download_image
+ENV OPENJDK17_URL
'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'
+ENV OPENJDK17_SHA256
'a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378'
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl --location --fail --silent --show-error --output
/tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \
+ echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c -
+
##
# build the final image
#
@@ -160,6 +167,14 @@ RUN mkdir -p /usr/lib/jvm && \
ln -s /usr/lib/jvm/java-11-adoptopenjdk /usr/lib/jvm/java-11 && \
rm /tmp/adoptopenjdk11.tar.gz
+# hadolint ignore=DL3010
+COPY --from=openjdk17_download_image /tmp/adoptopenjdk17.tar.gz
/tmp/adoptopenjdk17.tar.gz
+RUN mkdir -p /usr/lib/jvm && \
+ tar xzf /tmp/adoptopenjdk17.tar.gz -C /usr/lib/jvm && \
+ ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk17.tar.gz | head
-n1)")" /usr/lib/jvm/java-17-adoptopenjdk && \
+ ln -s /usr/lib/jvm/java-17-adoptopenjdk /usr/lib/jvm/java-17 && \
+ rm /tmp/adoptopenjdk17.tar.gz
+
# configure default environment for Yetus. Yetus in dockermode seems to require
# these values to be specified here; the various --foo-path flags do not
# propigate as expected, while these are honored.
diff --git a/pom.xml b/pom.xml
index e4a8c7b7671..77c752228ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
(in particular, if you are doing it for the first time), instead do
'mvn package'. If you are interested in the full story, see
https://issues.apache.org/jira/browse/HBASE-6795.
-
+for triggering test
-->
<modelVersion>4.0.0</modelVersion>
<parent>
