YETUS-378. flink, hbase, jmeter personalities custom tests need to be made qbt aware
Signed-off-by: Allen Wittenauer <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/b7c187fe Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/b7c187fe Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/b7c187fe Branch: refs/heads/YETUS-156 Commit: b7c187feac90370d1ce7a745d4b3c7b326ad138f Parents: 7501b40 Author: Allen Wittenauer <[email protected]> Authored: Mon Apr 18 14:18:15 2016 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Fri Apr 22 13:25:07 2016 -0700 ---------------------------------------------------------------------- precommit/personality/flink.sh | 61 +++++++------ precommit/personality/hbase.sh | 166 +++++++++++++++++++++++++++++++---- precommit/personality/jmeter.sh | 3 + 3 files changed, 180 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/b7c187fe/precommit/personality/flink.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/flink.sh b/precommit/personality/flink.sh index 6b4f0a8..76fa418 100755 --- a/precommit/personality/flink.sh +++ b/precommit/personality/flink.sh @@ -31,6 +31,10 @@ function personality_globals add_test_type flinklib +## @description flinklib file filter +## @audience private +## @stability evolving +## @param filename function flinklib_filefilter { local filename=$1 @@ -42,6 +46,9 @@ function flinklib_filefilter fi } +## @description count files in the lib dir +## @audience private +## @stability evolving function flinklib_count { find "${BASEDIR}" \ @@ -50,42 +57,46 @@ function flinklib_count | wc -l } -function flinklib_preapply +## @description check fliblib +## @audience private +## @stability evolving +## @param repostatus +function flinklib_rebuild { + declare repostatus=$1 + start_clock big_console_header "${PATCH_BRANCH} flink library dependencies" if ! verify_needed_test flinklib; then - echo "Patch does not need flinklib testing." + echo "${BUILDMODEMSG} does not need flinklib testing." return 0 fi pushd "${BASEDIR}" >/dev/null - echo_and_redirect "${PATCH_DIR}/branch-flinklib-root.txt" \ + echo_and_redirect "${PATCH_DIR}/${repostatus}-flinklib-root.txt" \ "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch if [[ $? != 0 ]]; then add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}." fi - FLINK_PRE_LIB_FILES=$(flinklib_count) - popd >/dev/null -} -function flinklib_postapply -{ - start_clock - big_console_header "Patch flink library dependencies" - - if ! verify_needed_test flinklib; then - echo "Patch does not need flinklib testing." + if [[ ${repostatus} = branch ]]; then + FLINK_PRE_LIB_FILES=$(flinklib_count) return 0 + else + FLINK_POST_LIB_FILES=$(flinklib_count) fi - - pushd "${BASEDIR}" >/dev/null - echo_and_redirect "${PATCH_DIR}/patch-flinklib-root.txt" \ - "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch - FLINK_POST_LIB_FILES=$(flinklib_count) popd >/dev/null + if [[ "${BUILDMODE}" = full ]]; then + if [[ ${FLINK_POST_LIB_FILES} -gt 0 ]]; then + add_vote_table -1 flinklib "Lib folder dependencies are currently ${FLINK_POST_LIB_FILES}" + return 1 + else + add_vote_table +1 flinklib "No lib folder dependencies!" + return 0 + fi + fi if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \ @@ -99,16 +110,4 @@ function flinklib_postapply "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))." fi return 0 -} - -function flinklib_rebuild -{ - declare repostatus=$1 - - if [[ "${repostatus}" = branch ]]; then - flinklib_preapply - else - flinklib_postinstall - fi -} - +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/b7c187fe/precommit/personality/hbase.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/hbase.sh b/precommit/personality/hbase.sh index e9e4ea8..5e6f0dc 100755 --- a/precommit/personality/hbase.sh +++ b/precommit/personality/hbase.sh @@ -22,17 +22,23 @@ personality_plugins "all" function personality_globals { #shellcheck disable=SC2034 + PROJECT_NAME=hbase + #shellcheck disable=SC2034 PATCH_BRANCH_DEFAULT=master #shellcheck disable=SC2034 JIRA_ISSUE_RE='^HBASE-[0-9]+$' #shellcheck disable=SC2034 GITHUB_REPO="apache/hbase" + # TODO use PATCH_BRANCH to select hadoop versions to use. # All supported Hadoop versions that we want to test the compilation with - HBASE_HADOOP_VERSIONS="2.4.1 2.5.2 2.6.0" + HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1" + + # TODO use PATCH_BRANCH to select jdk versions to use. # Override the maven options MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}" + } ## @description Queue up modules for this personality @@ -54,16 +60,19 @@ function personality_modules if [[ ${repostatus} == branch && ${testtype} == mvninstall ]] || - [[ "${BUILDMODE}" == full ]]; then - personality_enqueue_module . ${extra} - return - fi + [[ "${BUILDMODE}" == full ]];then + personality_enqueue_module . ${extra} + return + fi if [[ ${testtype} = findbugs ]]; then for module in "${CHANGED_MODULES[@]}"; do - # skip findbugs on hbase-shell + # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing + # in src/main/java where findbugs goes to look if [[ ${module} == hbase-shell ]]; then continue + elif [[ ${module} == hbase-it ]]; then + continue else # shellcheck disable=SC2086 personality_enqueue_module ${module} ${extra} @@ -72,6 +81,16 @@ function personality_modules return fi + if [[ ${testtype} = unit ]]; then + extra="${extra} -PrunAllTests" + + # Inject the jenkins build-id for our surefire invocations + # Used by zombie detection stuff, even though we're not including that yet. + if [ -n "${BUILD_ID}" ]; then + extra="${extra} -Dbuild.id=${BUILD_ID}" + fi + fi + for module in "${CHANGED_MODULES[@]}"; do # shellcheck disable=SC2086 personality_enqueue_module ${module} ${extra} @@ -79,9 +98,19 @@ function personality_modules } ################################################### +# Below here are our one-off tests specific to hbase. +# TODO break them into individual files so it's easier to maintain them? + +# TODO line length check? could ignore all java files since checkstyle gets them. + +################################################### add_test_type hadoopcheck +## @description hadoopcheck file filter +## @audience private +## @stability evolving +## @param filename function hadoopcheck_filefilter { local filename=$1 @@ -91,6 +120,10 @@ function hadoopcheck_filefilter fi } +## @description hadoopcheck test +## @audience private +## @stability evolving +## @param repostatus function hadoopcheck_rebuild { local repostatus=$1 @@ -114,7 +147,7 @@ function hadoopcheck_rebuild -Dhadoop-two.version="${hadoopver}" count=$(${GREP} -c ERROR "${logfile}") if [[ ${count} -gt 0 ]]; then - add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${hadoopver}." + add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}." ((result=result+1)) fi done @@ -123,14 +156,19 @@ function hadoopcheck_rebuild return 1 fi - add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}." + add_vote_table +1 hadoopcheck "${BUILDMODEMSG} does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}." return 0 } ###################################### +# TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol add_test_type hbaseprotoc +## @description hbaseprotoc file filter +## @audience private +## @stability evolving +## @param filename function hbaseprotoc_filefilter { local filename=$1 @@ -140,33 +178,38 @@ function hbaseprotoc_filefilter fi } +## @description hadoopcheck test +## @audience private +## @stability evolving +## @param repostatus function hbaseprotoc_rebuild { - local i=0 - local fn - local module - local logfile - local count - local result + declare repostatus=$1 + declare i=0 + declare fn + declare module + declare logfile + declare count + declare result if [[ "${repostatus}" = branch ]]; then return 0 fi - if ! verify_needed_test hbaseprotoc; then + verify_needed_test hbaseprotoc + if [[ $? == 0 ]]; then return 0 fi - big_console_header "Patch HBase protoc plugin" + big_console_header "HBase protoc plugin: ${BUILDMODE}" start_clock - personality_modules patch hbaseprotoc modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess # shellcheck disable=SC2153 - until [[ $i -eq ${#MODULE[@]} ]]; do + until [[ $i -eq "${#MODULE[@]}" ]]; do if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then ((result=result+1)) ((i=i+1)) @@ -197,6 +240,10 @@ function hbaseprotoc_rebuild add_test_type hbaseanti +## @description hbaseanti file filter +## @audience private +## @stability evolving +## @param filename function hbaseanti_filefilter { local filename=$1 @@ -206,13 +253,22 @@ function hbaseanti_filefilter fi } +## @description hbaseanti patch file check +## @audience private +## @stability evolving +## @param filename function hbaseanti_patchfile { local patchfile=$1 local warnings local result - if ! verify_needed_test hbaseanti; then + if [[ "${BUILDMODE}" = full ]]; then + return 0 + fi + + verify_needed_test hbaseanti + if [[ $? == 0 ]]; then return 0 fi @@ -239,3 +295,75 @@ function hbaseanti_patchfile add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns." return 0 } + + +## @description hbase custom mvnsite file filter. See HBASE-15042 +## @audience private +## @stability evolving +## @param filename +function mvnsite_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} = maven ]]; then + if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then + yetus_debug "tests/mvnsite: ${filename}" + add_test mvnsite + fi + fi +} + +## This is named so that yetus will check us right after running tests. +## Essentially, we check for normal failures and then we look for zombies. +#function hbase_unit_logfilter +#{ +# declare testtype="unit" +# declare input=$1 +# declare output=$2 +# declare processes +# declare process_output +# declare zombies +# declare zombie_count=0 +# declare zombie_process +# +# yetus_debug "in hbase-specific unit logfilter." +# +# # pass-through to whatever is counting actual failures +# if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then +# "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}" +# elif declare -f ${testtype}_logfilter >/dev/null; then +# "${testtype}_logfilter" "${input}" "${output}" +# fi +# +# start_clock +# if [ -n "${BUILD_ID}" ]; then +# yetus_debug "Checking for zombie test processes." +# processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}") +# if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then +# yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop." +# yetus_debug "${processes}" +# sleep 30 +# #shellcheck disable=SC2016 +# for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do +# # Test our zombie still running (and that it still an hbase build item) +# process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}") +# if [[ -n "${process_output}" ]]; then +# yetus_error "Zombie: ${process_output}" +# ((zombie_count = zombie_count + 1)) +# zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3) +# zombies="${zombies} ${zombie_process}" +# fi +# done +# fi +# if [ "${zombie_count}" -ne 0 ]; then +# add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)" +# populate_test_table "zombie unit tests" "${zombies}" +# else +# yetus_info "Zombie check complete. All test runs exited normally." +# stop_clock +# fi +# else +# add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies." +# fi +# +#} http://git-wip-us.apache.org/repos/asf/yetus/blob/b7c187fe/precommit/personality/jmeter.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/jmeter.sh b/precommit/personality/jmeter.sh index 194fc4f..b4e61b5 100755 --- a/precommit/personality/jmeter.sh +++ b/precommit/personality/jmeter.sh @@ -58,6 +58,9 @@ function jmeter_parse_args done } +## @description Download jmetere dependencies +## @audience private +## @stability evolving function jmeter_precheck { if [[ ${JMETER_DOWNLOAD_JARS} = true ]]; then
