Repository: hadoop Updated Branches: refs/heads/HADOOP-12111 bddc16c61 -> aa2d6e12d
HADOOP-12266. make test-patch plugins more consistent (Kengo Seki via aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa2d6e12 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa2d6e12 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa2d6e12 Branch: refs/heads/HADOOP-12111 Commit: aa2d6e12dc3c3be848e187da7eb03b077dccac5d Parents: bddc16c Author: Allen Wittenauer <[email protected]> Authored: Thu Jul 30 08:50:58 2015 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Thu Jul 30 08:50:58 2015 -0700 ---------------------------------------------------------------------- dev-support/test-patch.d/checkstyle.sh | 45 +-------------- dev-support/test-patch.d/pylint.sh | 62 +++------------------ dev-support/test-patch.d/shellcheck.sh | 85 +++++++---------------------- dev-support/test-patch.sh | 38 +++++++++++++ 4 files changed, 70 insertions(+), 160 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa2d6e12/dev-support/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh index f4bf79d..5538790 100755 --- a/dev-support/test-patch.d/checkstyle.sh +++ b/dev-support/test-patch.d/checkstyle.sh @@ -141,45 +141,6 @@ function checkstyle_preapply return 0 } -function checkstyle_calcdiffs -{ - local orig=$1 - local new=$2 - local diffout=$3 - local tmp=${PATCH_DIR}/cs.$$.${RANDOM} - local count=0 - local j - - # first, pull out just the errors - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch" - - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch" - - # compare the errors, generating a string of line - # numbers. Sorry portability: GNU diff makes this too easy - ${DIFF} --unchanged-line-format="" \ - --old-line-format="" \ - --new-line-format="%dn " \ - "${tmp}.branch" \ - "${tmp}.patch" > "${tmp}.lined" - - # now, pull out those lines of the raw output - # shellcheck disable=SC2013 - for j in $(cat "${tmp}.lined"); do - # shellcheck disable=SC2086 - head -${j} "${new}" | tail -1 >> "${diffout}" - done - - if [[ -f "${diffout}" ]]; then - # shellcheck disable=SC2016 - count=$(wc -l "${diffout}" | ${AWK} '{print $1}' ) - fi - rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null - echo "${count}" -} - function checkstyle_postapply { local result @@ -222,11 +183,9 @@ function checkstyle_postapply touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt" fi + calcdiffs "${PATCH_DIR}/branch-checkstyle-${fn}.txt" "${PATCH_DIR}/patch-checkstyle-${fn}.txt" > "${PATCH_DIR}/diff-checkstyle-${fn}.txt" #shellcheck disable=SC2016 - diffpostpatch=$(checkstyle_calcdiffs \ - "${PATCH_DIR}/branch-checkstyle-${fn}.txt" \ - "${PATCH_DIR}/patch-checkstyle-${fn}.txt" \ - "${PATCH_DIR}/diff-checkstyle-${fn}.txt" ) + diffpostpatch=$(wc -l "${PATCH_DIR}/diff-checkstyle-${fn}.txt" | ${AWK} '{print $1}') if [[ ${diffpostpatch} -gt 0 ]] ; then ((result = result + 1)) http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa2d6e12/dev-support/test-patch.d/pylint.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/pylint.sh b/dev-support/test-patch.d/pylint.sh index 8542dad..faa8136 100755 --- a/dev-support/test-patch.d/pylint.sh +++ b/dev-support/test-patch.d/pylint.sh @@ -71,7 +71,7 @@ function pylint_preapply for i in ${CHANGED_FILES}; do if [[ ${i} =~ \.py$ && -f ${i} ]]; then ${PYLINT} --indent-string=" " --output-format=parseable --reports=n "${i}" 2>/dev/null | - ${AWK} '1<NR' >> "${PATCH_DIR}/branchpylint-result.txt" + ${AWK} '1<NR' >> "${PATCH_DIR}/branch-pylint-result.txt" fi done popd >/dev/null @@ -80,49 +80,9 @@ function pylint_preapply return 0 } -function pylint_calcdiffs -{ - local orig=$1 - local new=$2 - local diffout=$3 - local tmp=${PATCH_DIR}/pl.$$.${RANDOM} - local count=0 - local j - - # first, pull out just the errors - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch" - - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch" - - # compare the errors, generating a string of line - # numbers. Sorry portability: GNU diff makes this too easy - ${DIFF} --unchanged-line-format="" \ - --old-line-format="" \ - --new-line-format="%dn " \ - "${tmp}.branch" \ - "${tmp}.patch" > "${tmp}.lined" - - # now, pull out those lines of the raw output - # shellcheck disable=SC2013 - for j in $(cat "${tmp}.lined"); do - # shellcheck disable=SC2086 - head -${j} "${new}" | tail -1 >> "${diffout}" - done - - if [[ -f "${diffout}" ]]; then - # shellcheck disable=SC2016 - count=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${diffout}") - fi - rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null - echo "${count}" -} - function pylint_postapply { local i - local msg local numPrepatch local numPostpatch local diffPostpatch @@ -152,32 +112,28 @@ function pylint_postapply for i in ${CHANGED_FILES}; do if [[ ${i} =~ \.py$ && -f ${i} ]]; then ${PYLINT} --indent-string=" " --output-format=parseable --reports=n "${i}" 2>/dev/null | - ${AWK} '1<NR' >> "${PATCH_DIR}/patchpylint-result.txt" + ${AWK} '1<NR' >> "${PATCH_DIR}/patch-pylint-result.txt" fi done popd >/dev/null # shellcheck disable=SC2016 PYLINT_VERSION=$(${PYLINT} --version 2>/dev/null | ${GREP} pylint | ${AWK} '{print $NF}') - PYLINT_VERSION=${PYLINT_VERSION%,} - msg="v${PYLINT_VERSION}" - add_footer_table pylint "${msg}" + add_footer_table pylint "v${PYLINT_VERSION%,}" - diffPostpatch=$(pylint_calcdiffs \ - "${PATCH_DIR}/branchpylint-result.txt" \ - "${PATCH_DIR}/patchpylint-result.txt" \ - "${PATCH_DIR}/diffpatchpylint.txt") + calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" "${PATCH_DIR}/patch-pylint-result.txt" > "${PATCH_DIR}/diff-patch-pylint.txt" + diffPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-pylint.txt") if [[ ${diffPostpatch} -gt 0 ]] ; then # shellcheck disable=SC2016 - numPrepatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branchpylint-result.txt") + numPrepatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-pylint-result.txt") # shellcheck disable=SC2016 - numPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patchpylint-result.txt") + numPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-pylint-result.txt") add_vote_table -1 pylint "The applied patch generated "\ - "${diffPostpatch} new pylint (v${PYLINT_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})." - add_footer_table pylint "@@BASE@@/diffpatchpylint.txt" + "${diffPostpatch} new pylint issues (total was ${numPrepatch}, now ${numPostpatch})." + add_footer_table pylint "@@BASE@@/diff-patch-pylint.txt" return 1 fi http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa2d6e12/dev-support/test-patch.d/shellcheck.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh index 0ccd1c0..14d1d18 100755 --- a/dev-support/test-patch.d/shellcheck.sh +++ b/dev-support/test-patch.d/shellcheck.sh @@ -64,7 +64,6 @@ function shellcheck_private_findbash function shellcheck_preapply { local i - local msg verify_needed_test shellcheck if [[ $? == 0 ]]; then @@ -80,21 +79,11 @@ function shellcheck_preapply start_clock - # shellcheck disable=SC2016 - SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}') - msg="v${SHELLCHECK_VERSION}" - - if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-5] ]]; then - msg="${msg} (This is an old version that has serious bugs. Consider upgrading.)" - fi - - add_footer_table shellcheck "${msg}" - echo "Running shellcheck against all identifiable shell scripts" pushd "${BASEDIR}" >/dev/null for i in $(shellcheck_private_findbash); do if [[ -f ${i} ]]; then - ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" + ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/branch-shellcheck-result.txt" fi done popd > /dev/null @@ -103,48 +92,10 @@ function shellcheck_preapply return 0 } -function shellcheck_calcdiffs -{ - local orig=$1 - local new=$2 - local diffout=$3 - local tmp=${PATCH_DIR}/sc.$$.${RANDOM} - local count=0 - local j - - # first, pull out just the errors - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch" - - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch" - - # compare the errors, generating a string of line - # numbers. Sorry portability: GNU diff makes this too easy - ${DIFF} --unchanged-line-format="" \ - --old-line-format="" \ - --new-line-format="%dn " \ - "${tmp}.branch" \ - "${tmp}.patch" > "${tmp}.lined" - - # now, pull out those lines of the raw output - # shellcheck disable=SC2013 - for j in $(cat "${tmp}.lined"); do - # shellcheck disable=SC2086 - head -${j} "${new}" | tail -1 >> "${diffout}" - done - - if [[ -f "${diffout}" ]]; then - # shellcheck disable=SC2016 - count=$(wc -l "${diffout}" | ${AWK} '{print $1}' ) - fi - rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null - echo "${count}" -} - function shellcheck_postapply { local i + local msg local numPrepatch local numPostpatch local diffPostpatch @@ -171,29 +122,35 @@ function shellcheck_postapply echo "Running shellcheck against all identifiable shell scripts" # we re-check this in case one has been added for i in $(shellcheck_private_findbash); do - ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt" + ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patch-shellcheck-result.txt" done - if [[ ! -f "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" ]]; then - touch "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" + if [[ ! -f "${PATCH_DIR}/branch-shellcheck-result.txt" ]]; then + touch "${PATCH_DIR}/branch-shellcheck-result.txt" fi # shellcheck disable=SC2016 - numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}') + SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}') + msg="v${SHELLCHECK_VERSION}" + if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-5] ]]; then + msg="${msg} (This is an old version that has serious bugs. Consider upgrading.)" + fi + add_footer_table shellcheck "${msg}" + calcdiffs "${PATCH_DIR}/branch-shellcheck-result.txt" "${PATCH_DIR}/patch-shellcheck-result.txt" > "${PATCH_DIR}/diff-patch-shellcheck.txt" # shellcheck disable=SC2016 - numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}') - - diffPostpatch=$(shellcheck_calcdiffs \ - "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \ - "${PATCH_DIR}/patchshellcheck-result.txt" \ - "${PATCH_DIR}/diffpatchshellcheck.txt" - ) + diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-shellcheck.txt" | ${AWK} '{print $1}') if [[ ${diffPostpatch} -gt 0 ]] ; then + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/branch-shellcheck-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patch-shellcheck-result.txt" | ${AWK} '{print $1}') + add_vote_table -1 shellcheck "The applied patch generated "\ - "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})." - add_footer_table shellcheck "@@BASE@@/diffpatchshellcheck.txt" + "${diffPostpatch} new shellcheck issues (total was ${numPrepatch}, now ${numPostpatch})." + add_footer_table shellcheck "@@BASE@@/diff-patch-shellcheck.txt" return 1 fi http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa2d6e12/dev-support/test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index c516beb..48f83a0 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -3433,6 +3433,44 @@ function add_bugsystem BUGSYSTEMS="${BUGSYSTEMS} $1" } +## @description Calculate the differences between the specified files +## @description and output it to stdout. +## @audience public +## @stability evolving +## @replaceable no +function calcdiffs +{ + local orig=$1 + local new=$2 + local tmp=${PATCH_DIR}/pl.$$.${RANDOM} + local count=0 + local j + + # first, pull out just the errors + # shellcheck disable=SC2016 + ${AWK} -F: '{print $NF}' "${orig}" > "${tmp}.branch" + + # shellcheck disable=SC2016 + ${AWK} -F: '{print $NF}' "${new}" > "${tmp}.patch" + + # compare the errors, generating a string of line + # numbers. Sorry portability: GNU diff makes this too easy + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${new}" | tail -1 + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + ############################################################################### ############################################################################### ###############################################################################
