Repository: incubator-ranger Updated Branches: refs/heads/master 839884094 -> 4a0d467d9
RANGER-209 Test Patch script for Ranger Signed-off-by: sneethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/4a0d467d Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/4a0d467d Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/4a0d467d Branch: refs/heads/master Commit: 4a0d467d92e000fab5ba75ff9782cd433d8f4e2f Parents: 8398840 Author: Ashish Singh <[email protected]> Authored: Wed Feb 4 14:27:55 2015 -0800 Committer: sneethiraj <[email protected]> Committed: Fri Feb 27 10:57:14 2015 -0800 ---------------------------------------------------------------------- dev-support/smart-apply-patch.sh | 145 ++++++ dev-support/test-patch.sh | 847 ++++++++++++++++++++++++++++++++++ 2 files changed, 992 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4a0d467d/dev-support/smart-apply-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh new file mode 100755 index 0000000..2a9dbd2 --- /dev/null +++ b/dev-support/smart-apply-patch.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# +# Determine if the patch file is a git diff file with prefixes. +# These files are generated via "git diff" *without* the --no-prefix option. +# +# We can apply these patches more easily because we know that the a/ and b/ +# prefixes in the "diff" lines stands for the project root directory. +# So we don't have to hunt for the project root. +# And of course, we know that the patch file was generated using git, so we +# know git apply can handle it properly. +# +# Arguments: file name. +# Return: 0 if it is a git diff; 1 otherwise. +# +is_git_diff_with_prefix() { + DIFF_TYPE="unknown" + while read -r line; do + if [[ "$line" =~ ^diff\ ]]; then + if [[ "$line" =~ ^diff\ \-\-git ]]; then + DIFF_TYPE="git" + else + return 1 # All diff lines must be diff --git lines. + fi + fi + if [[ "$line" =~ ^\+\+\+\ ]] || + [[ "$line" =~ ^\-\-\-\ ]]; then + if ! [[ "$line" =~ ^....[ab]/ ]]; then + return 1 # All +++ and --- lines must start with a/ or b/. + fi + fi + done < $1 + [ x$DIFF_TYPE == x"git" ] || return 1 + return 0 # return true (= 0 in bash) +} + +PATCH_FILE=$1 +DRY_RUN=$2 +if [ -z "$PATCH_FILE" ]; then + echo usage: $0 patch-file + exit 1 +fi + +PATCH=${PATCH:-patch} # allow overriding patch binary + +# Cleanup handler for temporary files +TOCLEAN="" +cleanup() { + rm $TOCLEAN + exit $1 +} +trap "cleanup 1" HUP INT QUIT TERM + +# Allow passing "-" for stdin patches +if [ "$PATCH_FILE" == "-" ]; then + PATCH_FILE=/tmp/tmp.in.$$ + cat /dev/fd/0 > $PATCH_FILE + TOCLEAN="$TOCLEAN $PATCH_FILE" +fi + +# Special case for git-diff patches without --no-prefix +if is_git_diff_with_prefix "$PATCH_FILE"; then + GIT_FLAGS="--binary -p1 -v" + if [[ -z $DRY_RUN ]]; then + GIT_FLAGS="$GIT_FLAGS --stat --apply " + echo Going to apply git patch with: git apply "${GIT_FLAGS}" + else + GIT_FLAGS="$GIT_FLAGS --check " + fi + git apply ${GIT_FLAGS} "${PATCH_FILE}" + exit $? +fi + +# Come up with a list of changed files into $TMP +TMP=/tmp/tmp.paths.$$ +TOCLEAN="$TOCLEAN $TMP" + +if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then + PLEVEL=0 + #if the patch applied at P0 there is the possability that all we are doing + # is adding new files and they would apply anywhere. So try to guess the + # correct place to put those files. + + TMP2=/tmp/tmp.paths.2.$$ + TOCLEAN="$TOCLEAN $TMP2" + + egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2 + + if [ ! -s $TMP2 ]; then + echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting." + cleanup 1 + fi + + #first off check that all of the files do not exist + FOUND_ANY=0 + for CHECK_FILE in $(cat $TMP2) + do + if [[ -f $CHECK_FILE ]]; then + FOUND_ANY=1 + fi + done + + if [[ "$FOUND_ANY" = "0" ]]; then + #all of the files are new files so we have to guess where the correct place to put it is. + + # if all of the lines start with a/ or b/, then this is a git patch that + # was generated without --no-prefix + if ! grep -qv '^a/\|^b/' $TMP2 ; then + echo Looks like this is a git patch. Stripping a/ and b/ prefixes + echo and incrementing PLEVEL + PLEVEL=$[$PLEVEL + 1] + sed -i -e 's,^[ab]/,,' $TMP2 + fi + fi +elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then + PLEVEL=1 +elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then + PLEVEL=2 +else + echo "The patch does not appear to apply with p0 to p2"; + cleanup 1; +fi + +# If this is a dry run then exit instead of applying the patch +if [[ -n $DRY_RUN ]]; then + cleanup 0; +fi + +echo Going to apply patch with: $PATCH -p$PLEVEL +$PATCH -p$PLEVEL -E < $PATCH_FILE + +cleanup $? http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4a0d467d/dev-support/test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh new file mode 100755 index 0000000..4995432 --- /dev/null +++ b/dev-support/test-patch.sh @@ -0,0 +1,847 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +#set -x +ulimit -n 1024 + +### Setup some variables. +### BUILD_URL is set by Hudson if it is run by patch process +### Read variables from properties file +bindir=$(dirname $0) + +# Defaults +if [ -z "$MAVEN_HOME" ]; then + MVN=mvn +else + MVN=$MAVEN_HOME/bin/mvn +fi + +PROJECT_NAME=Ranger +JENKINS=false +PATCH_DIR=/tmp +BASEDIR=$(pwd) +PS=${PS:-ps} +AWK=${AWK:-awk} +WGET=${WGET:-wget} +GIT=${GIT:-git} +GREP=${GREP:-grep} +PATCH=${PATCH:-patch} +DIFF=${DIFF:-diff} +JIRACLI=${JIRA:-jira} +FINDBUGS_HOME=${FINDBUGS_HOME} + +############################################################################### +printUsage() { + echo "Usage: $0 [options] patch-file | defect-number" + echo + echo "Where:" + echo " patch-file is a local patch file containing the changes to test" + echo " defect-number is a JIRA defect number (e.g. 'RANGER-1234') to test (Jenkins only)" + echo + echo "Options:" + echo "--patch-dir=<dir> The directory for working and output files (default '/tmp')" + echo "--basedir=<dir> The directory to apply the patch to (default current directory)" + echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" + echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')" + echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')" + echo "--git-cmd=<cmd> The 'git' command to use (default 'git')" + echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')" + echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')" + echo "--diff-cmd=<cmd> The 'diff' command to use (default 'diff')" + echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" + echo "--run-tests Run all tests below the base directory" + echo + echo "Jenkins-only options:" + echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" + echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')" + echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')" + echo "--jira-password=<pw> The password for the 'jira' command" +} + +############################################################################### +parseArgs() { + for i in $* + do + case $i in + --jenkins) + JENKINS=true + ;; + --patch-dir=*) + PATCH_DIR=${i#*=} + ;; + --basedir=*) + BASEDIR=${i#*=} + ;; + --mvn-cmd=*) + MVN=${i#*=} + ;; + --ps-cmd=*) + PS=${i#*=} + ;; + --awk-cmd=*) + AWK=${i#*=} + ;; + --wget-cmd=*) + WGET=${i#*=} + ;; + --git-cmd=*) + GIT=${i#*=} + ;; + --grep-cmd=*) + GREP=${i#*=} + ;; + --patch-cmd=*) + PATCH=${i#*=} + ;; + --diff-cmd=*) + DIFF=${i#*=} + ;; + --jira-cmd=*) + JIRACLI=${i#*=} + ;; + --jira-password=*) + JIRA_PASSWD=${i#*=} + ;; + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} + ;; + --dirty-workspace) + DIRTY_WORKSPACE=true + ;; + --run-tests) + RUN_TESTS=true + ;; + *) + PATCH_OR_DEFECT=$i + ;; + esac + done + if [ -z "$PATCH_OR_DEFECT" ]; then + printUsage + exit 1 + fi + if [[ $JENKINS == "true" ]] ; then + echo "Running in Jenkins mode" + defect=$PATCH_OR_DEFECT + else + echo "Running in developer mode" + JENKINS=false + ### PATCH_FILE contains the location of the patchfile + PATCH_FILE=$PATCH_OR_DEFECT + if [[ ! -e "$PATCH_FILE" ]] ; then + echo "Unable to locate the patch file $PATCH_FILE" + cleanupAndExit 0 + fi + ### Check if $PATCH_DIR exists. If it does not exist, create a new directory + if [[ ! -e "$PATCH_DIR" ]] ; then + mkdir "$PATCH_DIR" + if [[ $? == 0 ]] ; then + echo "$PATCH_DIR has been created" + else + echo "Unable to create $PATCH_DIR" + cleanupAndExit 0 + fi + fi + ### Obtain the patch filename to append it to the version number + defect=`basename $PATCH_FILE` + fi +} + +############################################################################### +checkout () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Testing patch for ${defect}." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + ### When run by a developer, if the workspace contains modifications, do not continue + ### unless the --dirty-workspace option was set + status=`$GIT status --porcelain` + if [[ $JENKINS == "false" ]] ; then + if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then + echo "ERROR: can't run in a workspace that contains the following modifications" + echo "$status" + cleanupAndExit 1 + fi + echo + else + cd $BASEDIR + $GIT reset --hard + $GIT clean -xdf + $GIT checkout master + $GIT pull --rebase + fi + GIT_REVISION=`git rev-parse --verify --short HEAD` + return $? +} + +############################################################################### +downloadPatch () { + ### Download latest patch file (ignoring .htm and .html) when run from patch process + if [[ $JENKINS == "true" ]] ; then + $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect + if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then + echo "$defect is not \"Patch Available\". Exiting." + cleanupAndExit 0 + fi + relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'` + patchURL="http://issues.apache.org${relativePatchURL}"; + patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'` + echo "$defect patch is being downloaded at `date` from" + echo "$patchURL" + $WGET -q -O $PATCH_DIR/patch $patchURL + VERSION=${GIT_REVISION}_${defect}_PATCH-${patchNum} + JIRA_COMMENT="Here are the results of testing the latest attachment + $patchURL + against master revision ${GIT_REVISION}." + + ### Copy the patch file to $PATCH_DIR + else + VERSION=PATCH-${defect} + cp $PATCH_FILE $PATCH_DIR/patch + if [[ $? == 0 ]] ; then + echo "Patch file $PATCH_FILE copied to $PATCH_DIR" + else + echo "Could not copy $PATCH_FILE to $PATCH_DIR" + cleanupAndExit 0 + fi + fi +} + +############################################################################### +verifyPatch () { + # Before building, check to make sure that the patch is valid + $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun + if [[ $? != 0 ]] ; then + echo "PATCH APPLICATION FAILED" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. The patch command could not apply the patch." + return 1 + else + return 0 + fi +} + +############################################################################### +prebuildWithoutPatch () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Pre-build master to verify master stability and javac warnings" + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "Compiling $(pwd)" + echo "$MVN clean test -DskipTests > $PATCH_DIR/masterJavacWarnings.txt 2>&1" + $MVN clean test -DskipTests > $PATCH_DIR/masterJavacWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "master compilation is broken?" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. master compilation may be broken." + return 1 + fi + + echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1" + $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "master javadoc compilation is broken?" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. master compilation may be broken." + return 1 + fi + + return 0 +} + +############################################################################### +### Check for @author tags in the patch +checkAuthor () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Checking there are no @author tags in the patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch` + echo "There appear to be $authorTags @author tags in the patch." + if [[ $authorTags != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Ranger community has agreed to not allow in code contributions." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 @author{color}. The patch does not contain any @author tags." + return 0 +} + +############################################################################### +### Check for tests in the patch +checkTests () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Checking there are new or changed tests in the patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch` + echo "There appear to be $testReferences test files referenced in the patch." + if [[ $testReferences == 0 ]] ; then + if [[ $JENKINS == "true" ]] ; then + patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` + if [[ $patchIsDoc != 0 ]] ; then + echo "The patch appears to be a documentation patch that doesn't require tests." + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests." + return 0 + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests. + Please justify why no new tests are needed for this patch. + Also please list what manual steps were performed to verify this patch." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files." + return 0 +} + +cleanUpXml () { + cd $BASEDIR/conf + for file in `ls *.xml.template` + do + rm -f `basename $file .template` + done + cd $BASEDIR +} + +############################################################################### +### Attempt to apply the patch +applyPatch () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Applying patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + export PATCH + $bindir/smart-apply-patch.sh $PATCH_DIR/patch + if [[ $? != 0 ]] ; then + echo "PATCH APPLICATION FAILED" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. The patch command could not apply the patch." + return 1 + fi + return 0 +} + +############################################################################### +calculateJavadocWarnings() { + WARNING_FILE="$1" + RET=$(egrep "^[0-9]+ warnings?$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') +} + +### Check there are no javadoc warnings +checkJavadocWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched javadoc warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" + $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 + calculateJavadocWarnings "$PATCH_DIR/masterJavadocWarnings.txt" + nummasterJavadocWarnings=$RET + calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" + numPatchJavadocWarnings=$RET + grep -i warning "$PATCH_DIR/masterJavadocWarnings.txt" > "$PATCH_DIR/masterJavadocWarningsFiltered.txt" + grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + diff -u "$PATCH_DIR/masterJavadocWarningsFiltered.txt" \ + "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ + "$PATCH_DIR/diffJavadocWarnings.txt" + rm -f "$PATCH_DIR/masterJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + echo "There appear to be $nummasterJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." + if [[ $nummasterJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then + if [[ $numPatchJavadocWarnings -gt $nummasterJavadocWarnings ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$nummasterJavadocWarnings))` warning messages. + See $BUILD_URL/artifact/patchprocess/diffJavadocWarnings.txt for details." + return 1 + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 javadoc{color}. There were no new javadoc warning messages." + return 0 +} + +############################################################################### +### Check there are no changes in the number of Javac warnings +checkJavacWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched javac warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean test -DskipTests > $PATCH_DIR/patchJavacWarnings.txt 2>&1" + $MVN clean test -DskipTests > $PATCH_DIR/patchJavacWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 javac{color:red}. The patch appears to cause the build to fail." + return 2 + fi + ### Compare master and patch javac warning numbers + if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then + $GREP '\[WARNING\]' $PATCH_DIR/masterJavacWarnings.txt > $PATCH_DIR/filteredmasterJavacWarnings.txt + $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt > $PATCH_DIR/filteredPatchJavacWarnings.txt + masterJavacWarnings=`cat $PATCH_DIR/filteredmasterJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + echo "There appear to be $masterJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." + if [[ $patchJavacWarnings != "" && $masterJavacWarnings != "" ]] ; then + if [[ $patchJavacWarnings -gt $masterJavacWarnings ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the master's current $masterJavacWarnings warnings)." + + $DIFF $PATCH_DIR/filteredmasterJavacWarnings.txt $PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt + JIRA_COMMENT_FOOTER="Javac warnings: $BUILD_URL/artifact/patchprocess/diffJavacWarnings.txt +$JIRA_COMMENT_FOOTER" + + return 1 + fi + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings." + return 0 +} + +############################################################################### +### Check there are no changes in the number of release audit (RAT) warnings +checkReleaseAuditWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched release audit warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1" + $MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1 + find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt + + ### Compare master and patch release audit warning numbers + if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then + patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` + echo "" + echo "" + echo "There appear to be $patchReleaseAuditWarnings release audit warnings after applying the patch." + if [[ $patchReleaseAuditWarnings != "" ]] ; then + if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings." + $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt + echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt + JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditProblems.txt +$JIRA_COMMENT_FOOTER" + return 1 + fi + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings." + return 0 +} + +############################################################################### +### Check there are no changes in the number of Checkstyle warnings +checkStyle () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched checkstyle warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN test checkstyle:checkstyle -DskipTests > $PATCH_DIR/patchStyleErrors.txt 2>&1" + $MVN test checkstyle:checkstyle -DskipTests > $PATCH_DIR/patchStyleErrors.txt 2>&1 + +# JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html +# $JIRA_COMMENT_FOOTER" + + ### calculate actual patchStyleErrors + patchStyleErrors=0 + if [[ -f target/munged/checkstyle-result.xml ]] ; then + $GREP -i '<error' target/munged/checkstyle-result.xml > $PATCH_DIR/filteredPatchCheckstyleWarnings.txt + patchStyleErrors=`cat $PATCH_DIR/filteredPatchCheckstyleWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + echo "" + echo "" + echo "There appear to be $patchStyleErrors checkstyle warnings after applying the patch." + fi + + if [[ $patchStyleErrors != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + -1 checkstyle. The patch generated $patchStyleErrors code style errors." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + +1 checkstyle. The patch generated 0 code style errors." + return 0 +} + +############################################################################### +### Install the new jars so tests and findbugs can find all of the updated jars +buildAndInstall () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Installing all of the jars" + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess" + $MVN install -Dmaven.javadoc.skip=true -DskipTests + return $? +} + + +############################################################################### +### Check there are no changes in the number of Findbugs warnings +checkFindbugsWarnings () { + findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version` + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched Findbugs warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + rc=0 + echo " Running findbugs " + echo "$MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1" + $MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1 + rc=$? + + if [ $rc != 0 ] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." + return 1 + fi + + findbugsWarnings=0 + for file in $(find $BASEDIR -name findbugsXml.xml) + do + relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix + if [ ! $relative_file == "target/findbugsXml.xml" ]; then + module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path + module_suffix=`basename ${module_suffix}` + fi + + cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` + echo "Found $newFindbugsWarnings Findbugs warnings ($file)" + findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) + $FINDBUGS_HOME/bin/convertXmlToText -html \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html + if [[ $newFindbugsWarnings > 0 ]] ; then + JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html +$JIRA_COMMENT_FOOTER" + fi + done + + if [[ $findbugsWarnings -gt 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." + return 0 +} + +############################################################################### +### Run the tests +runTests () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Running tests." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + failed_tests="" + failed_test_builds="" + test_timeouts="" + test_logfile=$PATCH_DIR/testrun.txt + echo " Running tests " + echo " $MVN clean install -fn -D${PROJECT_NAME}PatchProcess" + $MVN clean install -fae > $test_logfile 2>&1 + test_build_result=$? + cat $test_logfile + module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' $test_logfile` + if [[ -n "$module_test_timeouts" ]] ; then + test_timeouts="$test_timeouts $module_test_timeouts" + fi + module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"` + if [[ -n "$module_failed_tests" ]] ; then + failed_tests="${failed_tests} ${module_failed_tests}" + fi + if [[ $test_build_result != 0 && -z "$module_failed_tests" && -z "$module_test_timeouts" ]] ; then + failed_test_builds="$module $failed_test_builds" + fi + result=0 + comment_prefix=" {color:red}-1 core tests{color}." + if [[ -n "$failed_tests" ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + +$comment_prefix The patch failed these unit tests in $modules: +$failed_tests" + comment_prefix=" " + result=1 + fi + if [[ -n "$test_timeouts" ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + +$comment_prefix The following test timeouts occurred in $modules: +$test_timeouts" + comment_prefix=" " + result=1 + fi + if [[ -n "$failed_test_builds" ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + +$comment_prefix The test build failed in $failed_test_builds" + result=1 + fi + if [[ $result == 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 core tests{color}. The patch passed unit tests in $modules." + fi + return $result +} + +############################################################################### +# Find the maven module containing the given file. +findModule (){ + dir=`dirname $1` + while [ 1 ] + do + if [ -f "$dir/pom.xml" ] + then + echo $dir + return + else + dir=`dirname $dir` + fi + done +} + +############################################################################### +### Submit a comment to the defect's Jira +submitJiraComment () { + local result=$1 + ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer + if [[ $JENKINS == "false" ]] ; then + JIRA_COMMENT_FOOTER="" + fi + if [[ $result == 0 ]] ; then + comment="{color:green}+1 overall{color}. $JIRA_COMMENT + +$JIRA_COMMENT_FOOTER" + else + comment="{color:red}-1 overall{color}. $JIRA_COMMENT + +$JIRA_COMMENT_FOOTER" + fi + ### Output the test result to the console + echo " + + + +$comment" + + if [[ $JENKINS == "true" ]] ; then + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Adding comment to Jira." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + ### Update Jira with a comment + export USER=hudson + $JIRACLI -s https://issues.apache.org/jira -a addcomment -u hadoopqa -p $JIRA_PASSWD --comment "$comment" --issue $defect + $JIRACLI -s https://issues.apache.org/jira -a logout -u hadoopqa -p $JIRA_PASSWD + fi +} + +############################################################################### +### Cleanup files +cleanupAndExit () { + local result=$1 + if [[ $JENKINS == "true" ]] ; then + if [ -e "$PATCH_DIR" ] ; then + mv $PATCH_DIR $BASEDIR + fi + fi + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Finished build." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + exit $result +} + +############################################################################### +############################################################################### +############################################################################### + +JIRA_COMMENT="" +JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console + +This message is automatically generated." + +### Check if arguments to the script have been specified properly or not +parseArgs $@ +cd $BASEDIR + +checkout +RESULT=$? +if [[ $JENKINS == "true" ]] ; then + if [[ $RESULT != 0 ]] ; then + exit 100 + fi +fi +downloadPatch +verifyPatch +(( RESULT = RESULT + $? )) +if [[ $RESULT != 0 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +prebuildWithoutPatch +(( RESULT = RESULT + $? )) +if [[ $RESULT != 0 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +checkAuthor +(( RESULT = RESULT + $? )) + +if [[ $JENKINS == "true" ]] ; then + cleanUpXml +fi +checkTests +(( RESULT = RESULT + $? )) +applyPatch +APPLY_PATCH_RET=$? +(( RESULT = RESULT + $APPLY_PATCH_RET )) +if [[ $APPLY_PATCH_RET != 0 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +checkJavacWarnings +JAVAC_RET=$? +#2 is returned if the code could not compile +if [[ $JAVAC_RET == 2 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +(( RESULT = RESULT + $JAVAC_RET )) +checkJavadocWarnings +(( RESULT = RESULT + $? )) +checkStyle +(( RESULT = RESULT + $? )) +checkFindbugsWarnings +(( RESULT = RESULT + $? )) +checkReleaseAuditWarnings +(( RESULT = RESULT + $? )) +buildAndInstall +### Run tests for Jenkins or if explictly asked for by a developer +if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then + runTests + (( RESULT = RESULT + $? )) +fi +JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/ +$JIRA_COMMENT_FOOTER" + +submitJiraComment $RESULT +cleanupAndExit $RESULT
