This is an automated email from the ASF dual-hosted git repository. rajeshbabu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push: new 8a715fadd58 HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to ni… (#5689) 8a715fadd58 is described below commit 8a715fadd587dc89d5728090ba17cbad21a74398 Author: Rajeshbabu Chintaguntla <rajeshb...@apache.org> AuthorDate: Thu Feb 29 14:19:22 2024 +0530 HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to ni… (#5689) HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to nightly job HBASE-26372 [JDK17] Jenkins build support --- dev-support/Jenkinsfile | 119 +++++++++++++++++++++++++++++++++++ dev-support/Jenkinsfile_GitHub | 140 +++++++++++++++++++++++++++++++++++++++++ dev-support/docker/Dockerfile | 15 +++++ 3 files changed, 274 insertions(+) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 19256ccb9b1..461df28c8e9 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -37,6 +37,7 @@ pipeline { OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2' OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3' OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3' + OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3' PROJECT = 'hbase' PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' @@ -188,6 +189,7 @@ pipeline { stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match" stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match" stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match" + stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match" stash name: 'srctarball-result', allowEmpty: true, includes: "output-srctarball/doesn't-match" } } @@ -628,6 +630,123 @@ pipeline { } } } + + stage ('yetus jdk17 hadoop3 checks') { + when { + anyOf { + branch 'master';branch 'branch-3' + } + } + agent { + node { + label 'hbase' + } + } + environment { + BASEDIR = "${env.WORKSPACE}/component" + TESTS = "${env.DEEP_CHECKS}" + OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}" + OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}" + SET_JAVA_HOME = "/usr/lib/jvm/java-17" + // Activates hadoop 3.0 profile in maven runs. + HADOOP_PROFILE = '3.0' + SKIP_ERRORPRONE = true + } + steps { + // Must do prior to anything else, since if one of them timesout we'll stash the commentfile + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" + echo '(x) {color:red}-1 jdk17 hadoop3 checks{color}' >"${OUTPUT_DIR}/commentfile" + echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" + ''' + unstash 'yetus' + dir('component') { + checkout scm + } + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" + echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" + ls -lh "${OUTPUT_DIR_RELATIVE}/machine" + ''' + script { + def ret = sh( + returnStatus: true, + script: '''#!/usr/bin/env bash + set -e + declare -i status=0 + if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then + echo '(/) {color:green}+1 jdk17 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile" + else + echo '(x) {color:red}-1 jdk17 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile" + status=1 + fi + echo "-- For more information [see jdk17 report|${BUILD_URL}JDK17_20Nightly_20Build_20Report_20_28Hadoop3_29/]" >> "${OUTPUT_DIR}/commentfile" + exit "${status}" + ''' + ) + if (ret != 0) { + // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of + // test output. See HBASE-26339 for more details. + currentBuild.result = 'UNSTABLE' + } + } + } + post { + always { + stash name: 'jdk17-hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true + // zip surefire reports. + sh '''#!/bin/bash -e + if [ -d "${OUTPUT_DIR}/archiver" ]; then + count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l) + if [[ 0 -ne ${count} ]]; then + echo "zipping ${count} archived files" + zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver" + else + echo "No archived files, skipping compressing." + fi + else + echo "No archiver directory, skipping compressing." + fi + ''' + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip" + ) + ] + ) + ]) + // remove the big test logs zip file, store the nightlies url in test_logs.html + sh '''#!/bin/bash -e + if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then + echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" + rm -rf "${OUTPUT_DIR}/test_logs.zip" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + else + echo "No test_logs.zip, skipping" + fi + ''' + // Has to be relative to WORKSPACE. + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" + publishHTML target: [ + allowMissing : true, + keepAll : true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE. + reportDir : "${env.OUTPUT_DIR_RELATIVE}", + reportFiles : 'console-report.html', + reportName : 'JDK17 Nightly Build Report (Hadoop3)' + ] + } + } + } + // This is meant to mimic what a release manager will do to create RCs. // See http://hbase.apache.org/book.html#maven.release // TODO (HBASE-23870): replace this with invocation of the release tool diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub index bf916903411..62b16287e1e 100644 --- a/dev-support/Jenkinsfile_GitHub +++ b/dev-support/Jenkinsfile_GitHub @@ -55,6 +55,7 @@ pipeline { WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check' WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check' WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check' + WORKDIR_REL_JDK17_HADOOP3_CHECK = 'yetus-jdk17-hadoop3-check' ASF_NIGHTLIES = 'https://nightlies.apache.org' ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" @@ -470,6 +471,145 @@ pipeline { } } } + stage ('yetus jdk17 hadoop3 checks') { + agent { + node { + label 'hbase' + } + } + environment { + // customized per parallel stage + PLUGINS = "${JDK_SPECIFIC_PLUGINS}" + SET_JAVA_HOME = '/usr/lib/jvm/java-17' + WORKDIR_REL = "${WORKDIR_REL_JDK17_HADOOP3_CHECK}" + // identical for all parallel stages + WORKDIR = "${WORKSPACE}/${WORKDIR_REL}" + YETUSDIR = "${WORKDIR}/${YETUS_REL}" + SOURCEDIR = "${WORKDIR}/${SRC_REL}" + PATCHDIR = "${WORKDIR}/${PATCH_REL}" + BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}" + DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}" + YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}" + SKIP_ERRORPRONE = true + } + when { + allOf { + // this will return true if the pipeline is building a change request, such as a GitHub pull request. + changeRequest() + expression { env.CHANGE_TARGET in ['master', 'branch-3'] } + } + } + steps { + dir("${SOURCEDIR}") { + checkout scm + } + dir("${YETUSDIR}") { + checkout([ + $class : 'GitSCM', + branches : [[name: "${YETUS_VERSION}"]], + userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]] + ) + } + dir("${WORKDIR}") { + withCredentials([ + usernamePassword( + credentialsId: 'apache-hbase-at-github.com', + passwordVariable: 'GITHUB_PASSWORD', + usernameVariable: 'GITHUB_USER' + )]) { + script { + def ret = sh( + label: 'test-patch', + returnStatus: true, + script: '''#!/bin/bash -e + hostname -a ; pwd ; ls -la + printenv 2>&1 | sort + echo "[INFO] Launching Yetus via ${YETUS_DRIVER}" + "${YETUS_DRIVER}" + ''' + ) + if (ret != 0) { + // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of + // test output. See HBASE-26339 for more details. + currentBuild.result = 'UNSTABLE' + } + } + } + } + } + post { + always { + junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml", + allowEmptyResults: true, skipPublishingChecks: true + sh label: 'zip surefire reports', script: '''#!/bin/bash -e + if [ -d "${PATCHDIR}/archiver" ]; then + count=$(find "${PATCHDIR}/archiver" -type f | wc -l) + if [[ 0 -ne ${count} ]]; then + echo "zipping ${count} archived files" + zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver" + else + echo "No archived files, skipping compressing." + fi + else + echo "No archiver directory, skipping compressing." + fi + ''' + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip" + ) + ] + ) + ]) + // remove the big test logs zip file, store the nightlies url in test_logs.txt + sh '''#!/bin/bash -e + if [ -f "${PATCHDIR}/test_logs.zip" ]; then + echo "Remove ${PATCHDIR}/test_logs.zip for saving space" + rm -rf "${PATCHDIR}/test_logs.zip" + python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" + else + echo "No test_logs.zip, skipping" + fi + ''' + // Has to be relative to WORKSPACE. + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit" + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*" + publishHTML target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE + reportDir: "${WORKDIR_REL}/${PATCH_REL}", + reportFiles: 'report.html', + reportName: 'PR JDK17 Hadoop3 Check Report' + ] + } + // Jenkins pipeline jobs fill slaves on PRs without this :( + cleanup() { + script { + sh label: 'Cleanup workspace', script: '''#!/bin/bash -e + # See YETUS-764 + if [ -f "${PATCHDIR}/pidfile.txt" ]; then + echo "test-patch process appears to still be running: killing" + kill `cat "${PATCHDIR}/pidfile.txt"` || true + sleep 10 + fi + if [ -f "${PATCHDIR}/cidfile.txt" ]; then + echo "test-patch container appears to still be running: killing" + docker kill `cat "${PATCHDIR}/cidfile.txt"` || true + fi + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + dir ("${WORKDIR}") { + deleteDir() + } + } + } + } + } } } } diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 1f841d1d867..dcd84c89c21 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -110,6 +110,13 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \ echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c - +FROM base_image AS openjdk17_download_image +ENV OPENJDK17_URL 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz' +ENV OPENJDK17_SHA256 'a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \ + echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c - + ## # build the final image # @@ -158,6 +165,14 @@ RUN mkdir -p /usr/lib/jvm && \ ln -s /usr/lib/jvm/java-11-adoptopenjdk /usr/lib/jvm/java-11 && \ rm /tmp/adoptopenjdk11.tar.gz +# hadolint ignore=DL3010 +COPY --from=openjdk17_download_image /tmp/adoptopenjdk17.tar.gz /tmp/adoptopenjdk17.tar.gz +RUN mkdir -p /usr/lib/jvm && \ + tar xzf /tmp/adoptopenjdk17.tar.gz -C /usr/lib/jvm && \ + ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk17.tar.gz | head -n1)")" /usr/lib/jvm/java-17-adoptopenjdk && \ + ln -s /usr/lib/jvm/java-17-adoptopenjdk /usr/lib/jvm/java-17 && \ + rm /tmp/adoptopenjdk17.tar.gz + # configure default environment for Yetus. Yetus in dockermode seems to require # these values to be specified here; the various --foo-path flags do not # propigate as expected, while these are honored.