Repository: hbase Updated Branches: refs/heads/master bcaf2fd20 -> a4272a9f0
HBASE-19570 Add hadoop3 tests to Nightly master/branch-2 runs. Jenkins fails the whole build immediately if any stage fails. Hadoop2 tests run before Hadoop3 tests. So Hadoop3 tests will run only if hadoop2 tests pass. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a4272a9f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a4272a9f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a4272a9f Branch: refs/heads/master Commit: a4272a9f08f1005d1d3b2511dfbe8eca577f0d49 Parents: bcaf2fd Author: Apekshit Sharma <a...@apache.org> Authored: Thu Dec 21 11:20:40 2017 -0800 Committer: Apekshit Sharma <a...@apache.org> Committed: Thu Dec 21 17:44:04 2017 -0800 ---------------------------------------------------------------------- dev-support/Jenkinsfile | 73 ++++++++++++++++++++++++++++++--- dev-support/hbase-personality.sh | 8 ++++ dev-support/hbase_nightly_yetus.sh | 5 +++ 3 files changed, 80 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/a4272a9f/dev-support/Jenkinsfile ---------------------------------------------------------------------- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 744e9b0..dcef649 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -35,7 +35,7 @@ pipeline { BASEDIR = "${env.WORKSPACE}/component" YETUS_RELEASE = '0.6.0' PROJECT = 'hbase' - PROJET_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' + PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag. AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh' WHITESPACE_IGNORE_LIST = '.*/generated/.*' @@ -102,7 +102,7 @@ fi dir ("${env.TOOLS}") { sh """#!/usr/bin/env bash echo "Downloading Project personality." -curl -L -o personality.sh "${env.PROJET_PERSONALITY}" +curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" """ } stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh" @@ -203,15 +203,15 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}" } } } - stage ('yetus jdk8 checks') { + stage ('yetus jdk8 hadoop2 checks') { when { not { branch 'branch-1.1*' } } environment { - TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout' - OUTPUT_DIR_RELATIVE = "output-jdk8" + TESTS = 'mvninstall,compile,javac,unit,htmlout' + OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop2" OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}" // This isn't strictly needed on branches that only support jdk8, but doesn't hurt // and is needed on branches that do both jdk7 and jdk8 @@ -253,7 +253,68 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}" // Has to be relative to WORKSPACE. reportDir : "${env.OUTPUT_DIR_RELATIVE}", reportFiles : 'console-report.html', - reportName : 'JDK8 Nightly Build Report' + reportName : 'JDK8 Nightly Build Report (Hadoop2)' + ] + } + } + } + stage ('yetus jdk8 hadoop3 checks') { + when { + not { + branch 'branch-1*' + } + } + environment { + // Failure in any stage fails the build and consecutive stages are not built. + // Findbugs is part of this last yetus stage to prevent findbugs precluding hadoop3 + // tests. + TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout' + OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop3" + OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}" + // This isn't strictly needed on branches that only support jdk8, but doesn't hurt + // and is needed on branches that do both jdk7 and jdk8 + SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64' + // Activates hadoop 3.0 profile in maven runs. + HADOOP_PROFILE = '3.0' + } + steps { + unstash 'yetus' + sh '''#!/usr/bin/env bash + rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" + rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" +''' + sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh" + } + post { + always { + // Not sure how two junit test reports will work. Disabling this for now. + // junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true + // zip surefire reports. + sh '''#!/bin/bash -e + if [ -d "${OUTPUT_DIR}/archiver" ]; then + count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l) + if [[ 0 -ne ${count} ]]; then + echo "zipping ${count} archived files" + zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver" + else + echo "No archived files, skipping compressing." + fi + else + echo "No archiver directory, skipping compressing." + fi +''' + // Has to be relative to WORKSPACE. + archive "${env.OUTPUT_DIR_RELATIVE}/*" + archive "${env.OUTPUT_DIR_RELATIVE}/**/*" + publishHTML target: [ + allowMissing : true, + keepAll : true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE. + reportDir : "${env.OUTPUT_DIR_RELATIVE}", + reportFiles : 'console-report.html', + reportName : 'JDK8 Nightly Build Report (Hadoop3)' ] } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a4272a9f/dev-support/hbase-personality.sh ---------------------------------------------------------------------- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index cf5eacd..e24930d 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -81,6 +81,9 @@ function personality_parse_args --include-tests-url=*) INCLUDE_TESTS_URL=${i#*=} ;; + --hadoop-profile=*) + HADOOP_PROFILE=${i#*=} + ;; esac done } @@ -103,6 +106,10 @@ function personality_modules extra="-DHBasePatchProcess" + if [[ -n "${HADOOP_PROFILE}" ]]; then + extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}" + fi + # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on # full source code instead. In this case, do full compiles, tests, etc instead of per # module. @@ -115,6 +122,7 @@ function personality_modules fi if [[ ${testtype} == mvninstall ]]; then + # shellcheck disable=SC2086 personality_enqueue_module . ${extra} return fi http://git-wip-us.apache.org/repos/asf/hbase/blob/a4272a9f/dev-support/hbase_nightly_yetus.sh ---------------------------------------------------------------------- diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index d37292b..e1175d2 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -76,6 +76,11 @@ if [[ -n "${INCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}") fi +# For testing with specific hadoop version. Activates corresponding profile in maven runs. +if [[ -n "${HADOOP_PROFILE}" ]]; then + YETUS_ARGS=("--hadoop-profile=${HADOOP_PROFILE}" "${YETUS_ARGS[@]}") +fi + if [[ true == "${DEBUG}" ]]; then YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}") fi