HADOOP-12113. update test-patch branch to latest code (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09a2e360 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09a2e360 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09a2e360 Branch: refs/heads/HADOOP-12111 Commit: 09a2e360f2603f54cddf22e2bec6a0f1d77d596b Parents: 214ac3e Author: Allen Wittenauer <[email protected]> Authored: Fri Jun 26 23:30:37 2015 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Fri Jun 26 23:30:37 2015 -0700 ---------------------------------------------------------------------- dev-support/smart-apply-patch.sh | 4 +- dev-support/test-patch.d/checkstyle.sh | 211 +- dev-support/test-patch.d/shellcheck.sh | 25 +- dev-support/test-patch.d/whitespace.sh | 35 +- dev-support/test-patch.sh | 2722 +++++++++++------- .../hadoop-common/CHANGES-HADOOP-12111.txt | 2 + 6 files changed, 1837 insertions(+), 1162 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a2e360/dev-support/smart-apply-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh index ebcb660..ddfd940 100755 --- a/dev-support/smart-apply-patch.sh +++ b/dev-support/smart-apply-patch.sh @@ -43,7 +43,9 @@ PATCH=${PATCH:-patch} # allow overriding patch binary # Cleanup handler for temporary files TOCLEAN="" cleanup() { - rm $TOCLEAN + if [[ -n ${TOCLEAN} ]]; then + rm $TOCLEAN + fi exit $1 } trap "cleanup 1" HUP INT QUIT TERM http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a2e360/dev-support/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh index 6311584..1fbe88e 100755 --- a/dev-support/test-patch.d/checkstyle.sh +++ b/dev-support/test-patch.d/checkstyle.sh @@ -18,79 +18,126 @@ add_plugin checkstyle CHECKSTYLE_TIMER=0 -# if it ends in an explicit .sh, then this is shell code. -# if it doesn't have an extension, we assume it is shell code too function checkstyle_filefilter { local filename=$1 - if [[ ${filename} =~ \.java$ ]]; then - add_test checkstyle + if [[ ${BUILDTOOL} == maven + || ${BUILDTOOL} == ant ]]; then + if [[ ${filename} =~ \.java$ ]]; then + add_test checkstyle + fi fi } -function checkstyle_mvnrunner +function checkstyle_runner { - local logfile=$1 - local output=$2 + local repostatus=$1 local tmp=${PATCH_DIR}/$$.${RANDOM} local j + local i=0 + local fn + local savestart=${TIMER} + local savestop + local output + local logfile + local repo + local modulesuffix + local cmd + + modules_reset + + if [[ ${repostatus} == branch ]]; then + repo=${PATCH_BRANCH} + else + repo="the patch" + fi + + #shellcheck disable=SC2153 + until [[ $i -eq ${#MODULE[@]} ]]; do + start_clock + fn=$(module_file_fragment "${MODULE[${i}]}") + modulesuffix=$(basename "${MODULE[${i}]}") + output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt" + logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt" + pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null + + case ${BUILDTOOL} in + maven) + cmd="${MVN} ${MAVEN_ARGS[*]} clean test \ + checkstyle:checkstyle \ + -Dcheckstyle.consoleOutput=true \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch" + ;; + ant) + cmd="${ANT} \ + -Dcheckstyle.consoleOutput=true \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \ + ${ANT_ARGS[*]} checkstyle" + ;; + esac + + #shellcheck disable=SC2086 + echo ${cmd} "> ${logfile}" + #shellcheck disable=SC2086 + ${cmd} 2>&1 \ + | tee "${logfile}" \ + | ${GREP} ^/ \ + | ${SED} -e "s,${BASEDIR},.,g" \ + > "${tmp}" + + if [[ $? == 0 ]] ; then + module_status ${i} +1 "${logfile}" "${modulesuffix} in ${repo} passed checkstyle" + else + module_status ${i} -1 "${logfile}" "${modulesuffix} in ${repo} failed checkstyle" + ((result = result + 1)) + fi + savestop=$(stop_clock) + #shellcheck disable=SC2034 + MODULE_STATUS_TIMER[${i}]=${savestop} + + for j in ${CHANGED_FILES}; do + ${GREP} "${j}" "${tmp}" >> "${output}" + done - "${MVN}" clean test checkstyle:checkstyle -DskipTests \ - -Dcheckstyle.consoleOutput=true \ - "-D${PROJECT_NAME}PatchProcess" 2>&1 \ - | tee "${logfile}" \ - | ${GREP} ^/ \ - | ${SED} -e "s,${BASEDIR},.,g" \ - > "${tmp}" - - # the checkstyle output files are massive, so - # let's reduce the work by filtering out files - # that weren't changed. Some modules are - # MASSIVE and this can cut the output down to - # by orders of magnitude!! - for j in ${CHANGED_FILES}; do - ${GREP} "${j}" "${tmp}" >> "${output}" + rm "${tmp}" 2>/dev/null + # shellcheck disable=SC2086 + popd >/dev/null + ((i=i+1)) done - rm "${tmp}" 2>/dev/null + TIMER=${savestart} + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 } function checkstyle_preapply { - local module_suffix - local modules=${CHANGED_MODULES} - local module + local result - verify_needed_test checkstyle + big_console_header "${PATCH_BRANCH} checkstyle" + + start_clock + verify_needed_test checkstyle if [[ $? == 0 ]]; then + echo "Patch does not need checkstyle testing." return 0 fi - big_console_header "checkstyle plugin: prepatch" - - start_clock - - for module in ${modules} - do - pushd "${module}" >/dev/null - echo " Running checkstyle in ${module}" - module_suffix=$(basename "${module}") - - checkstyle_mvnrunner \ - "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \ - "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" - - if [[ $? != 0 ]] ; then - echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?" - add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken." - fi - popd >/dev/null - done + personality_modules branch checkstyle + checkstyle_runner branch + result=$? + modules_messages branch checkstyle true # keep track of how much as elapsed for us already CHECKSTYLE_TIMER=$(stop_clock) + if [[ ${result} != 0 ]]; then + return 1 + fi return 0 } @@ -135,71 +182,71 @@ function checkstyle_calcdiffs function checkstyle_postapply { - local rc=0 + local result local module - local modules=${CHANGED_MODULES} - local module_suffix + local fn + local i=0 local numprepatch=0 local numpostpatch=0 local diffpostpatch=0 - verify_needed_test checkstyle + big_console_header "Patch checkstyle plugin" + start_clock + + verify_needed_test checkstyle if [[ $? == 0 ]]; then + echo "Patch does not need checkstyle testing." return 0 fi - big_console_header "checkstyle plugin: postpatch" + personality_modules patch checkstyle + checkstyle_runner patch + result=$? - start_clock # add our previous elapsed to our new timer # by setting the clock back offset_clock "${CHECKSTYLE_TIMER}" - for module in ${modules} - do - pushd "${module}" >/dev/null - echo " Running checkstyle in ${module}" - module_suffix=$(basename "${module}") - - checkstyle_mvnrunner \ - "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \ - "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" - - if [[ $? != 0 ]] ; then - ((rc = rc +1)) - echo "Post-patch checkstyle compilation is broken." - add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken." + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) continue fi + module=${MODULE[$i]} + fn=$(module_file_fragment "${module}") + + if [[ ! -f "${PATCH_DIR}/branch-checkstyle-${fn}.txt" ]]; then + touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt" + fi #shellcheck disable=SC2016 diffpostpatch=$(checkstyle_calcdiffs \ - "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \ - "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \ - "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" ) + "${PATCH_DIR}/branch-checkstyle-${fn}.txt" \ + "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \ + "${PATCH_DIR}/diff-checkstyle-${fn}.txt" ) if [[ ${diffpostpatch} -gt 0 ]] ; then - ((rc = rc + 1)) + ((result = result + 1)) # shellcheck disable=SC2016 - numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}') + numprepatch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') # shellcheck disable=SC2016 - numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}') + numpostpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') - add_jira_table -1 checkstyle "The applied patch generated "\ - "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})." - footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt" + module_status ${i} -1 "diff-checkstyle-${fn}.txt" "Patch generated "\ + "${diffpostpatch} new checkstyle issues in "\ + "${module} (total was ${numprepatch}, now ${numpostpatch})." fi - - popd >/dev/null + ((i=i+1)) done - if [[ ${rc} -gt 0 ]] ; then - add_jira_footer checkstyle "${footer}" + modules_messages patch checkstyle true + + if [[ ${result} != 0 ]]; then return 1 fi - add_jira_table +1 checkstyle "There were no new checkstyle issues." return 0 -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a2e360/dev-support/test-patch.d/shellcheck.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh index 9277ea5..1c06a5d 100755 --- a/dev-support/test-patch.d/shellcheck.sh +++ b/dev-support/test-patch.d/shellcheck.sh @@ -56,6 +56,7 @@ function shellcheck_private_findbash function shellcheck_preapply { local i + local msg verify_needed_test shellcheck if [[ $? == 0 ]]; then @@ -65,7 +66,7 @@ function shellcheck_preapply big_console_header "shellcheck plugin: prepatch" if [[ ! -x "${SHELLCHECK}" ]]; then - hadoop_error "shellcheck is not available." + yetus_error "shellcheck is not available." return 0 fi @@ -73,6 +74,13 @@ function shellcheck_preapply # shellcheck disable=SC2016 SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}') + msg="v${SHELLCHECK_VERSION}" + + if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-5] ]]; then + msg="${msg} (This is an old version that has serious bugs. Consider upgrading.)" + fi + + add_footer_table shellcheck "${msg}" echo "Running shellcheck against all identifiable shell scripts" pushd "${BASEDIR}" >/dev/null @@ -138,8 +146,8 @@ function shellcheck_postapply big_console_header "shellcheck plugin: postpatch" if [[ ! -x "${SHELLCHECK}" ]]; then - hadoop_error "shellcheck is not available." - add_jira_table 0 shellcheck "Shellcheck was not available." + yetus_error "shellcheck is not available." + add_vote_table 0 shellcheck "Shellcheck was not available." return 0 fi @@ -155,8 +163,13 @@ function shellcheck_postapply ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt" done + if [[ ! -f "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" ]]; then + touch "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" + fi + # shellcheck disable=SC2016 numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}') + # shellcheck disable=SC2016 numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}') @@ -167,12 +180,12 @@ function shellcheck_postapply ) if [[ ${diffPostpatch} -gt 0 ]] ; then - add_jira_table -1 shellcheck "The applied patch generated "\ + add_vote_table -1 shellcheck "The applied patch generated "\ "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})." - add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt" + add_footer_table shellcheck "@@BASE@@/diffpatchshellcheck.txt" return 1 fi - add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues." + add_vote_table +1 shellcheck "There were no new shellcheck issues." return 0 } http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a2e360/dev-support/test-patch.d/whitespace.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh index 324481c..6fc033b 100755 --- a/dev-support/test-patch.d/whitespace.sh +++ b/dev-support/test-patch.d/whitespace.sh @@ -19,28 +19,47 @@ add_plugin whitespace function whitespace_postapply { local count - local j + local result=0 big_console_header "Checking for whitespace at the end of lines" start_clock pushd "${BASEDIR}" >/dev/null - for j in ${CHANGED_FILES}; do - ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt" - done + # shellcheck disable=SC2016 + ${AWK} '/\t/ {print $0}' \ + "${GITDIFFCONTENT}" \ + | ${GREP} -v Makefile: >> "${PATCH_DIR}/whitespace-tabs.txt" + + ${GREP} -E '[[:blank:]]$' \ + "${GITDIFFCONTENT}" \ + >> "${PATCH_DIR}/whitespace-eol.txt" # shellcheck disable=SC2016 - count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}') + count=$(wc -l "${PATCH_DIR}/whitespace-eol.txt" | ${AWK} '{print $1}') if [[ ${count} -gt 0 ]]; then - add_jira_table -1 whitespace "The patch has ${count}"\ + add_vote_table -1 whitespace "The patch has ${count}"\ " line(s) that end in whitespace. Use git apply --whitespace=fix." - add_jira_footer whitespace "@@BASE@@/whitespace.txt" + add_footer_table whitespace "@@BASE@@/whitespace-eol.txt" + ((result=result+1)) + fi + + # shellcheck disable=SC2016 + count=$(wc -l "${PATCH_DIR}/whitespace-tabs.txt" | ${AWK} '{print $1}') + + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 whitespace "The patch has ${count}"\ + " line(s) with tabs." + add_footer_table whitespace "@@BASE@@/whitespace-tabs.txt" + ((result=result+1)) + fi + + if [[ ${result} -gt 0 ]]; then popd >/dev/null return 1 fi popd >/dev/null - add_jira_table +1 whitespace "The patch has no lines that end in whitespace." + add_vote_table +1 whitespace "Patch has no whitespace issues." return 0 }
