Repository: oozie
Updated Branches:
  refs/heads/master bd5445280 -> 51920be84


OOZIE-1986 Add FindBugs report to pre-commit build (andras.piros via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/51920be8
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/51920be8
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/51920be8

Branch: refs/heads/master
Commit: 51920be8480c80638e11faff20ad089b02a7f5fb
Parents: bd54452
Author: Robert Kanter <rkan...@cloudera.com>
Authored: Tue Oct 18 18:27:20 2016 -0700
Committer: Robert Kanter <rkan...@cloudera.com>
Committed: Tue Oct 18 18:27:20 2016 -0700

----------------------------------------------------------------------
 bin/test-patch-11-findbugs-diff                 | 274 +++++++++++++++++++
 ...est-patch-11-findbugs-diff-0.1.0-all.jar.md5 |   1 +
 pom.xml                                         |   1 +
 release-log.txt                                 |   1 +
 4 files changed, 277 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/bin/test-patch-11-findbugs-diff
----------------------------------------------------------------------
diff --git a/bin/test-patch-11-findbugs-diff b/bin/test-patch-11-findbugs-diff
new file mode 100644
index 0000000..0376ab0
--- /dev/null
+++ b/bin/test-patch-11-findbugs-diff
@@ -0,0 +1,274 @@
+#!/bin/bash
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+if [ "${TESTPATCHDEBUG}" == "true" ] ; then
+  set -x
+fi
+
+BASEDIR=$(pwd)
+TASKNAME="FINDBUGS_DIFF"
+OP=""
+TEMPDIR=${BASEDIR}/tmp
+REPORTDIR=""
+SUMMARYFILE=""
+STDOUT="/dev/null"
+MVNPASSTHRU=""
+FINDBUGS_JAR_URL=https://repo1.maven.org/maven2/me/andrz/findbugs/findbugs-diff/0.1.0/findbugs-diff-0.1.0-all.jar
+FINDBUGS_JAR=findbugs-diff-0.1.0-all.jar
+# Priorities.HIGH_PRIORITY and Priorities.NORMAL_PRIORITY
+FINDBUGS_PRIORITY_THRESHOLD=2
+# Scariest and scary
+FINDBUGS_RANK_THRESHOLD=9
+FINDBUGS_XML_NAME=findbugsXml.xml
+
+
+cleanup_and_exit() {
+  remove_file_if_present "${DIFF_DIR}/${FINDBUGS_JAR}"
+  remove_file_if_present "${DIFF_DIR}/${FINDBUGS_JAR}.md5"
+
+  exit "$1"
+}
+
+
+remove_file_if_present() {
+  FILE_NAME=$1
+
+  if [ -f "${FILE_NAME}" ]; then
+    rm -f "${FILE_NAME}"
+    echo "[TRACE] File [${FILE_NAME}] removed"
+  fi
+}
+
+print_usage() {
+  echo "Usage: $0 --taskname | (--op=pre|post|report --tempdir=<TEMP DIR> 
--reportdir=<REPORT DIR> --summaryfile=<SUMMARY FILE>) [--verbose] 
[-D<VALUE>...] [-P<VALUE>...]"
+  echo
+}
+
+
+parse_args() {
+  for i in "$@"; do
+    case $i in
+    --taskname)
+      echo ${TASKNAME}
+      exit 0
+      ;;
+    --op=*)
+      OP=${i#*=}
+      ;;
+    --tempdir=*)
+      TEMPDIR=${i#*=}
+      ;;
+    --reportdir=*)
+      REPORTDIR=${i#*=}
+      ;;
+    --summaryfile=*)
+      SUMMARYFILE=${i#*=}
+      ;;
+    --verbose)
+      STDOUT="/dev/stdout"
+      ;;
+    -D*)
+      MVNPASSTHRU="${MVNPASSTHRU} $i"
+      ;;
+    -P*)
+      MVNPASSTHRU="${MVNPASSTHRU} $i"
+      ;;
+    esac
+  done
+  if [[ "${TASKNAME}" == "" || "${OP}" == "" || "${TEMPDIR}" == "" || 
"${REPORTDIR}" == "" || "${SUMMARYFILE}" == "" ]] ; then
+    echo "Missing options"
+    echo
+    print_usage
+    cleanup_and_exit 1
+  fi
+  if [[ "${OP}" != "pre" && "${OP}" != "post" && "${OP}" != "report" ]] ; then
+    echo "Invalid operation"
+    echo
+    print_usage
+    cleanup_and_exit 1
+  fi
+}
+
+
+verify_and_save_findbugs_output() {
+  FINDBUGS_OUTPUT_DIR=$1
+  BRANCH_LABEL=$2
+  REPORT_SUFFIX=$3
+
+  echo "[TRACE] Verifying and saving FindBugs output in ${BRANCH_LABEL}"
+
+  mvn clean verify -DskipTests "${MVNPASSTHRU}" | tee 
"${REPORTDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt" >> ${STDOUT}
+
+  if [ ! -d "$FINDBUGS_OUTPUT_DIR" ]; then
+    mkdir -p "${FINDBUGS_OUTPUT_DIR}"
+  fi
+
+  find . -name ${FINDBUGS_XML_NAME} -exec rsync -avqR {} 
"${FINDBUGS_OUTPUT_DIR}" ";"
+
+  if [ "$(ls -A "${FINDBUGS_OUTPUT_DIR}")" ] ; then
+    echo "{color:green}+1{color} ${BRANCH_LABEL} produces FindBugs output" >> 
"${TEMPDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt"
+  else
+    echo "{color:red}-1{color} ${BRANCH_LABEL} does not produce FindBugs 
output" >> "${TEMPDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt"
+  fi
+
+  echo "[TRACE] FindBugs output in ${BRANCH_LABEL} verified and saved"
+}
+
+
+download_and_check_findbugs_diff_jar() {
+  DIFF_DIR=$1
+  BASH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+  mkdir -p "${DIFF_DIR}"
+
+  echo "[TRACE] Downloading FindBugs diff JAR from ${FINDBUGS_JAR_URL}"
+
+  curl -Ls ${FINDBUGS_JAR_URL} > "${DIFF_DIR}/${FINDBUGS_JAR}"
+
+  echo "[TRACE] FindBugs diff JAR downloaded"
+
+  if hash md5 2>/dev/null; then
+    md5 -q "${DIFF_DIR}/${FINDBUGS_JAR}" > "${DIFF_DIR}/${FINDBUGS_JAR}.md5"
+  elif hash md5sum 2>/dev/null; then
+    md5sum "${DIFF_DIR}/${FINDBUGS_JAR}" > "${DIFF_DIR}/${FINDBUGS_JAR}.md5"
+  else
+    echo "[ERROR] Neither md5 nor md5sum are present, cannot check FindBugs 
diff JAR"
+    echo "{color:red}-1{color} Neither md5 nor md5sum are present, cannot 
check FindBugs diff JAR." >> "${SUMMARYFILE}"
+    cleanup_and_exit 1
+  fi
+
+  jarMd5DiffCount=$(grep -Fxvf "${BASH_DIR}/test-patch-11-${FINDBUGS_JAR}.md5" 
"${DIFF_DIR}/${FINDBUGS_JAR}.md5" | wc -l)
+
+  if [ ${jarMd5DiffCount} -gt "0" ]; then
+    echo "[ERROR] FindBugs diff JAR has a weird MD5 sum, rejecting"
+    echo "{color:red}-1{color} FindBugs diff JAR has a weird MD5 sum, 
rejecting." >> "${SUMMARYFILE}"
+    cleanup_and_exit 1
+  fi
+
+  echo "[TRACE] FindBugs diff JAR checked, is safe to use"
+
+}
+
+
+perform_findbugs_diffs() {
+  PRE_DIR=$1
+  POST_DIR=$2
+  DIFF_DIR=$3
+
+  echo "[TRACE] Performing FindBugs diffs"
+
+  preFindbugsDiffs=()
+  while IFS= read -r -d '' fn; do
+    preFindbugsDiffs+=(${fn})
+  done <   <(find "${PRE_DIR}" -name ${FINDBUGS_XML_NAME} -print0)
+
+  postFindbugsDiffs=()
+  while IFS= read -r -d '' fn; do
+    postFindbugsDiffs+=(${fn})
+  done <   <(find "${POST_DIR}" -name ${FINDBUGS_XML_NAME} -print0)
+
+  for index in "${!preFindbugsDiffs[@]}"; do
+    preFindbugsDir=${preFindbugsDiffs[index]}
+    diffDirPostfix=${preFindbugsDir/*${TASKNAME}/${TASKNAME}}
+    diffDirPostfix=${diffDirPostfix:18}
+    diffDirPostfix=${diffDirPostfix%%/target/findbugs/findbugsXml.xml}
+
+    java -jar -Dorg.slf4j.simpleLogger.defaultLogLevel=ERROR 
"${DIFF_DIR}/${FINDBUGS_JAR}" \
+        --outDir="${DIFF_DIR}/${diffDirPostfix}" "${preFindbugsDiffs[index]}" 
"${postFindbugsDiffs[index]}"
+  done
+
+  echo "[TRACE] FindBugs diffs performed"
+}
+
+
+check_findbugs_diffs_and_create_reports() {
+  DIFF_DIR=$1
+  REPORT=()
+
+  echo "[TRACE] Checking FindBugs diffs and creating reports"
+
+  belowThresholdCount=0
+  totalCount=0
+  while IFS= read -r -d '' fn; do
+    componentDir=${fn/*${TASKNAME}/${TASKNAME}}
+    componentDir=${componentDir:19}
+    htmlFileName=${componentDir%%.xml}.html
+    componentDir=${componentDir%%/findbugs-new.xml}
+
+    newBugTotalCount=$(xmllint --xpath "count(/BugCollection/BugInstance)" 
"${fn}")
+    newBugBelowThresholdCount=$(xmllint --xpath 
"count(/BugCollection/BugInstance[@priority <= ${FINDBUGS_PRIORITY_THRESHOLD} 
or @rank <= ${FINDBUGS_RANK_THRESHOLD}])" "${fn}")
+
+    belowThresholdCount=$((belowThresholdCount + newBugBelowThresholdCount))
+    totalCount=$((totalCount + newBugTotalCount))
+
+    if [ "${newBugBelowThresholdCount}" -gt "0" ]; then
+      echo "[ERROR] There are [${newBugBelowThresholdCount}] new bugs found 
below threshold in [${componentDir}]."
+      REPORT+=("{color:red}-1{color} There are [${newBugBelowThresholdCount}] 
new bugs found below threshold in [${componentDir}] that must be fixed.")
+      echo "[DEBUG] You can find the FindBugs diff here (look for the red and 
orange ones): ${htmlFileName}"
+      REPORT+=("You can find the FindBugs diff here (look for the red and 
orange ones): ${htmlFileName}")
+      REPORT+=("The most important FindBugs errors are:")
+      while IFS= read -r line; do
+        REPORT+=( "${line}" );
+      done < <(echo cat "/BugCollection/BugInstance[@priority <= 
${FINDBUGS_PRIORITY_THRESHOLD} or @rank <= 
${FINDBUGS_RANK_THRESHOLD}]/SourceLine/Message/text() | 
/BugCollection/BugInstance[@priority <= 2 or @rank <= 9]/LongMessage/text()" \
+      | xmllint --shell "${fn}" | grep -v '\-\-\-\-\-\-\-' | grep -v '/ >' | 
sed -e 'N;s/\(.*\)\n\(.*\)/\2: \1/')
+    elif [ "${newBugTotalCount}" -gt "0" ]; then
+      echo "[WARN] There are [${newBugTotalCount}] new bugs found in 
[${componentDir}]."
+      REPORT+=("{color:orange}0{color} There are [${newBugTotalCount}] new 
bugs found in [${componentDir}] that would be nice to have fixed.")
+      echo "[DEBUG] You can find the FindBugs diff here: ${htmlFileName}"
+      REPORT+=("You can find the FindBugs diff here: ${htmlFileName}")
+    else
+      echo "[DEBUG] There are no new bugs found in [${componentDir}]."
+      REPORT+=("{color:green}+1{color} There are no new bugs found in 
[${componentDir}].")
+    fi
+
+  done <   <(find "${DIFF_DIR}" -name findbugs-new.xml -print0)
+
+  if [ "${belowThresholdCount}" -gt "0" ]; then
+    echo "[ERROR] There are [${belowThresholdCount}] new bugs found below 
threshold in total that must be fixed."
+    echo "{color:red}-1{color} There are [${belowThresholdCount}] new bugs 
found below threshold in total that must be fixed." \
+        >> "${SUMMARYFILE}"
+  elif [ "${totalCount}" -gt "0" ]; then
+    echo "[WARN] There are [${totalCount}] new bugs found in total that would 
be nice to have fixed."
+    echo "{color:orange}0{color} There are [${totalCount}] new bugs found in 
total that would be nice to have fixed." \
+        >> "${SUMMARYFILE}"
+  else
+    echo "[INFO] There are no new bugs found totally]."
+    echo "{color:green}+1{color} There are no new bugs found in total." >> 
"${SUMMARYFILE}"
+  fi
+
+  for line in "${REPORT[@]}" ; do
+    echo ".    ${line}" >> "${SUMMARYFILE}"
+  done
+
+  echo "[TRACE] FindBugs diffs checked and reports created"
+}
+
+
+parse_args "$@"
+
+case ${OP} in
+  pre)
+    verify_and_save_findbugs_output "${TEMPDIR}/${TASKNAME}/pre" HEAD clean
+    ;;
+  post)
+    verify_and_save_findbugs_output "${TEMPDIR}/${TASKNAME}/post" PATCH patch
+    ;;
+  report)
+    download_and_check_findbugs_diff_jar "${TEMPDIR}/${TASKNAME}/diff"
+    perform_findbugs_diffs "${TEMPDIR}/${TASKNAME}/pre" 
"${TEMPDIR}/${TASKNAME}/post" "${TEMPDIR}/${TASKNAME}/diff"
+    check_findbugs_diffs_and_create_reports "${TEMPDIR}/${TASKNAME}/diff"
+    ;;
+esac
+
+cleanup_and_exit 0

http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5
----------------------------------------------------------------------
diff --git a/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5 
b/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5
new file mode 100644
index 0000000..28179b6
--- /dev/null
+++ b/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5
@@ -0,0 +1 @@
+d965fab3dbf678189924ad6d61fc7a3d

http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 83a1d42..fcd023c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1826,6 +1826,7 @@
                         <exclude>**/.idea/**</exclude>
                         <exclude>*.patch</exclude>
                         <exclude>**/*.json</exclude>
+                        <exclude>bin/test-patch-*</exclude>
                     </excludes>
                 </configuration>
             </plugin>

http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f73fbb8..855c01d 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 4.4.0 release (trunk - unreleased)
 
+OOZIE-1986 Add FindBugs report to pre-commit build (andras.piros via rkanter)
 OOZIE-2634 Queue dump command message is confusing when the queue is empty 
(andras.piros via rkanter)
 
 

Reply via email to