HADOOP-11746. rewrite test-patch.sh (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/640ffeae Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/640ffeae Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/640ffeae Branch: refs/heads/YARN-2928 Commit: 640ffeae04d985ef92bbaddee4c51b5e0e7b085e Parents: 39e7929 Author: Allen Wittenauer <[email protected]> Authored: Tue Apr 21 21:29:45 2015 +0100 Committer: Zhijie Shen <[email protected]> Committed: Tue Apr 21 16:16:56 2015 -0700 ---------------------------------------------------------------------- dev-support/shelldocs.py | 31 +- dev-support/test-patch.d/checkstyle.sh | 149 + dev-support/test-patch.d/shellcheck.sh | 138 + dev-support/test-patch.d/whitespace.sh | 40 + dev-support/test-patch.sh | 2839 +++++++++++++----- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + 6 files changed, 2430 insertions(+), 769 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ffeae/dev-support/shelldocs.py ---------------------------------------------------------------------- diff --git a/dev-support/shelldocs.py b/dev-support/shelldocs.py index 2547450..fc7601a 100755 --- a/dev-support/shelldocs.py +++ b/dev-support/shelldocs.py @@ -17,6 +17,26 @@ import sys import string from optparse import OptionParser +asflicense=''' +<!--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--> +''' + def docstrip(key,string): string=re.sub("^## @%s " % key ,"",string) string=string.lstrip() @@ -220,17 +240,18 @@ def main(): funcdef.addreturn(line) elif line.startswith('function'): funcdef.setname(line) - if options.skipprnorep: - if funcdef.getaudience() == "Private" and \ - funcdef.getreplace() == "No": + if options.skipprnorep and \ + funcdef.getaudience() == "Private" and \ + funcdef.getreplace() == "No": pass - else: - allfuncs.append(funcdef) + else: + allfuncs.append(funcdef) funcdef=ShellFunction() allfuncs=sorted(allfuncs) outfile=open(options.outfile, "w") + outfile.write(asflicense) for line in toc(allfuncs): outfile.write(line) http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ffeae/dev-support/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh new file mode 100755 index 0000000..460709e --- /dev/null +++ b/dev-support/test-patch.d/checkstyle.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_plugin checkstyle + +CHECKSTYLE_TIMER=0 + +# if it ends in an explicit .sh, then this is shell code. +# if it doesn't have an extension, we assume it is shell code too +function checkstyle_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + add_test checkstyle + fi +} + +function checkstyle_preapply +{ + verify_needed_test checkstyle + + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "checkstyle plugin: prepatch" + + start_clock + echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}checkstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess" + if [[ $? != 0 ]] ; then + echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?" + add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} checkstyle compilation may be broken." + return 1 + fi + + cp -p "${BASEDIR}/target/checkstyle-result.xml" \ + "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" + + # keep track of how much as elapsed for us already + CHECKSTYLE_TIMER=$(stop_clock) + return 0 +} + +function checkstyle_postapply +{ + verify_needed_test checkstyle + + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "checkstyle plugin: postpatch" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${CHECKSTYLE_TIMER}" + + echo_and_redirect "${PATCH_DIR}/patchcheckstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess" + if [[ $? != 0 ]] ; then + echo "Post-patch checkstyle compilation is broken." + add_jira_table -1 checkstyle "Post-patch checkstyle compilation is broken." + return 1 + fi + + cp -p "${BASEDIR}/target/checkstyle-result.xml" \ + "${PATCH_DIR}/checkstyle-result-patch.xml" + + checkstyle_runcomparison + + # shellcheck disable=SC2016 + CHECKSTYLE_POSTPATCH=$(wc -l "${PATCH_DIR}/checkstyle-result-diff.txt" | ${AWK} '{print $1}') + + if [[ ${CHECKSTYLE_POSTPATCH} -gt 0 ]] ; then + + add_jira_table -1 checkstyle "The applied patch generated "\ + "${CHECKSTYLE_POSTPATCH}" \ + " additional checkstyle issues." + add_jira_footer checkstyle "@@BASE@@/checkstyle-result-diff.txt" + + return 1 + fi + add_jira_table +1 checkstyle "There were no new checkstyle issues." + return 0 +} + + +function checkstyle_runcomparison +{ + + python <(cat <<EOF +import os +import sys +import xml.etree.ElementTree as etree +from collections import defaultdict + +if len(sys.argv) != 3 : + print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0] + exit(1) + +def path_key(x): + path = x.attrib['name'] + return path[path.find('${PROJECT_NAME}-'):] + +def print_row(path, master_errors, patch_errors): + print '%s\t%s\t%s' % (k,master_dict[k],child_errors) + +master = etree.parse(sys.argv[1]) +patch = etree.parse(sys.argv[2]) + +master_dict = defaultdict(int) + +for child in master.getroot().getchildren(): + if child.tag != 'file': + continue + child_errors = len(child.getchildren()) + if child_errors == 0: + continue + master_dict[path_key(child)] = child_errors + +for child in patch.getroot().getchildren(): + if child.tag != 'file': + continue + child_errors = len(child.getchildren()) + if child_errors == 0: + continue + k = path_key(child) + if child_errors > master_dict[k]: + print_row(k, master_dict[k], child_errors) + +EOF +) "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" "${PATCH_DIR}/checkstyle-result-patch.xml" > "${PATCH_DIR}/checkstyle-result-diff.txt" + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ffeae/dev-support/test-patch.d/shellcheck.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh new file mode 100755 index 0000000..b60da36 --- /dev/null +++ b/dev-support/test-patch.d/shellcheck.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_plugin shellcheck + +SHELLCHECK_TIMER=0 + +SHELLCHECK=${SHELLCHECK:-$(which shellcheck)} + +SHELLCHECK_SPECIFICFILES="" + +# if it ends in an explicit .sh, then this is shell code. +# if it doesn't have an extension, we assume it is shell code too +function shellcheck_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.sh$ ]]; then + add_test shellcheck + SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}" + fi + + if [[ ! ${filename} =~ \. ]]; then + add_test shellcheck + fi +} + +function shellcheck_private_findbash +{ + local i + local value + local list + + while read line; do + value=$(find "${line}" ! -name '*.cmd' -type f \ + | ${GREP} -E -v '(.orig$|.rej$)') + list="${list} ${value}" + done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d) + # shellcheck disable=SC2086 + echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u +} + +function shellcheck_preapply +{ + local i + + verify_needed_test shellcheck + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "shellcheck plugin: prepatch" + + if [[ -z "${SHELLCHECK}" ]]; then + hadoop_error "shellcheck is not available." + fi + + start_clock + + # shellcheck disable=SC2016 + SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}') + + echo "Running shellcheck against all identifiable shell scripts" + pushd "${BASEDIR}" >/dev/null + for i in $(shellcheck_private_findbash); do + if [[ -f ${i} ]]; then + ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" + fi + done + popd > /dev/null + # keep track of how much as elapsed for us already + SHELLCHECK_TIMER=$(stop_clock) + return 0 +} + +function shellcheck_postapply +{ + local i + + verify_needed_test shellcheck + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "shellcheck plugin: postpatch" + + if [[ -z "${SHELLCHECK}" ]]; then + hadoop_error "shellcheck is not available." + fi + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${SHELLCHECK_TIMER}" + + echo "Running shellcheck against all identifiable shell scripts" + # we re-check this in case one has been added + for i in $(shellcheck_private_findbash); do + ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt" + done + + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}') + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}') + + ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \ + "${PATCH_DIR}/patchshellcheck-result.txt" \ + | ${GREP} '^+\.' \ + > "${PATCH_DIR}/diffpatchshellcheck.txt" + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diffpatchshellcheck.txt" | ${AWK} '{print $1}') + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_jira_table -1 shellcheck "The applied patch generated "\ + "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})." + add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt" + return 1 + fi + + add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues." + return 0 +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/640ffeae/dev-support/test-patch.d/whitespace.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh new file mode 100755 index 0000000..deac654 --- /dev/null +++ b/dev-support/test-patch.d/whitespace.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_plugin whitespace + +function whitespace_preapply +{ + local count + + big_console_header "Checking for whitespace at the end of lines" + start_clock + + ${GREP} '^+' "${PATCH_DIR}/patch" | ${GREP} '[[:blank:]]$' > "${PATCH_DIR}/whitespace.txt" + + # shellcheck disable=SC2016 + count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}') + + if [[ ${count} -gt 0 ]]; then + add_jira_table -1 whitespace "The patch has ${count}"\ + " line(s) that end in whitespace." + add_jira_footer whitespace "@@BASE@@/whitespace.txt" + return 1 + fi + + add_jira_table +1 whitespace "The patch has no lines that end in whitespace." + return 0 +}
