Repository: hbase Updated Branches: refs/heads/HBASE-19491 [created] 7e656e352
HBASE-19570 Add hadoop3 tests to Nightly master/branch-2 runs Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7e656e35 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7e656e35 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7e656e35 Branch: refs/heads/HBASE-19491 Commit: 7e656e352e47b8b7a3c5dbfe0dbc963c7d54e1a5 Parents: 6cefabe Author: Apekshit Sharma <a...@apache.org> Authored: Thu Dec 21 11:20:40 2017 -0800 Committer: Apekshit Sharma <a...@apache.org> Committed: Thu Dec 21 11:20:40 2017 -0800 ---------------------------------------------------------------------- dev-support/Jenkinsfile | 75 +++++++++++++++++++++++++++++++-- dev-support/hbase-personality.sh | 7 +++ dev-support/hbase_nightly_yetus.sh | 5 +++ 3 files changed, 83 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/7e656e35/dev-support/Jenkinsfile ---------------------------------------------------------------------- diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 744e9b0..e197cb0 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -35,7 +35,8 @@ pipeline { BASEDIR = "${env.WORKSPACE}/component" YETUS_RELEASE = '0.6.0' PROJECT = 'hbase' - PROJET_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' + // todo: revert + PROJET_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/HBASE-19491/dev-support/hbase-personality.sh' // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag. AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh' WHITESPACE_IGNORE_LIST = '.*/generated/.*' @@ -109,6 +110,12 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}" } } stage ('yetus general check') { + // todo: revert + when { + not { + branch 'HBASE-19491' + } + } environment { // TODO does hadoopcheck need to be jdk specific? // Should be things that work with multijdk @@ -203,19 +210,79 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}" } } } - stage ('yetus jdk8 checks') { + stage ('yetus jdk8 hadoop2 checks') { when { not { branch 'branch-1.1*' } } environment { + TESTS = 'mvninstall,compile,javac,unit,htmlout' + OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop2" + OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}" + // This isn't strictly needed on branches that only support jdk8, but doesn't hurt + // and is needed on branches that do both jdk7 and jdk8 + SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64' + } + steps { + unstash 'yetus' + sh '''#!/usr/bin/env bash + rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" + rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" +''' + sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh" + } + post { + always { + junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true + // zip surefire reports. + sh '''#!/bin/bash -e + if [ -d "${OUTPUT_DIR}/archiver" ]; then + count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l) + if [[ 0 -ne ${count} ]]; then + echo "zipping ${count} archived files" + zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver" + else + echo "No archived files, skipping compressing." + fi + else + echo "No archiver directory, skipping compressing." + fi +''' + // Has to be relative to WORKSPACE. + archive "${env.OUTPUT_DIR_RELATIVE}/*" + archive "${env.OUTPUT_DIR_RELATIVE}/**/*" + publishHTML target: [ + allowMissing : true, + keepAll : true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE. + reportDir : "${env.OUTPUT_DIR_RELATIVE}", + reportFiles : 'console-report.html', + reportName : 'JDK8 Nightly Build Report (Hadoop2)' + ] + } + } + } + stage ('yetus jdk8 hadoop3 checks') { + when { + not { + branch 'branch-1*' + } + } + environment { + // Failure in any stage fails the build and consecutive stages are not built. + // Findbugs is part of this last yetus stage to prevent findbugs precluding hadoop3 + // tests. TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout' - OUTPUT_DIR_RELATIVE = "output-jdk8" + OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop3" OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}" // This isn't strictly needed on branches that only support jdk8, but doesn't hurt // and is needed on branches that do both jdk7 and jdk8 SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64' + // Activates hadoop 3.0 profile in maven runs. + HADOOP_PROFILE = '3.0' } steps { unstash 'yetus' @@ -253,7 +320,7 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}" // Has to be relative to WORKSPACE. reportDir : "${env.OUTPUT_DIR_RELATIVE}", reportFiles : 'console-report.html', - reportName : 'JDK8 Nightly Build Report' + reportName : 'JDK8 Nightly Build Report (Hadoop3)' ] } } http://git-wip-us.apache.org/repos/asf/hbase/blob/7e656e35/dev-support/hbase-personality.sh ---------------------------------------------------------------------- diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index cf5eacd..7a2a838 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -81,6 +81,9 @@ function personality_parse_args --include-tests-url=*) INCLUDE_TESTS_URL=${i#*=} ;; + --hadoop-profile=*) + HADOOP_PROFILE=${i#*=} + ;; esac done } @@ -103,6 +106,10 @@ function personality_modules extra="-DHBasePatchProcess" + if [[ -n "${HADOOP_PROFILE}" ]]; then + extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}" + fi + # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on # full source code instead. In this case, do full compiles, tests, etc instead of per # module. http://git-wip-us.apache.org/repos/asf/hbase/blob/7e656e35/dev-support/hbase_nightly_yetus.sh ---------------------------------------------------------------------- diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index d37292b..e1175d2 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -76,6 +76,11 @@ if [[ -n "${INCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}") fi +# For testing with specific hadoop version. Activates corresponding profile in maven runs. +if [[ -n "${HADOOP_PROFILE}" ]]; then + YETUS_ARGS=("--hadoop-profile=${HADOOP_PROFILE}" "${YETUS_ARGS[@]}") +fi + if [[ true == "${DEBUG}" ]]; then YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}") fi