Repository: yetus Updated Branches: refs/heads/master ef0e1c058 -> dfd03bdc6
YETUS-390. qbt needs an abbreviated email mode Signed-off-by: Marco Zuehlke <[email protected]> Signed-off-by: Chris Nauroth <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/dfd03bdc Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/dfd03bdc Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/dfd03bdc Branch: refs/heads/master Commit: dfd03bdc68a469ab522a3d517a7e47858f97e0bb Parents: ef0e1c0 Author: Allen Wittenauer <[email protected]> Authored: Sun May 29 15:51:23 2016 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Fri Jun 3 16:25:07 2016 -0700 ---------------------------------------------------------------------- .../documentation/in-progress/precommit-qbt.md | 7 +- precommit/core.d/builtin-bugsystem.sh | 4 +- precommit/test-patch.d/briefreport.sh | 254 +++++++++++++++++++ precommit/test-patch.d/htmlout.sh | 9 +- precommit/test-patch.d/jira.sh | 7 +- precommit/test-patch.sh | 72 +++--- 6 files changed, 308 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/asf-site-src/source/documentation/in-progress/precommit-qbt.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/in-progress/precommit-qbt.md b/asf-site-src/source/documentation/in-progress/precommit-qbt.md index 3e8f578..badaf60 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-qbt.md +++ b/asf-site-src/source/documentation/in-progress/precommit-qbt.md @@ -61,4 +61,9 @@ ${FILE,path="<report-file-path>"} </body></html> ``` -NOTE: Be aware that ASF mailing lists do not allow HTML formatted email. +If your mailing lists do not allow HTML-formatted email, then the `--brief-report-file` +provides a solution. This option creates a very plain, reduced content text file +suitable for email. It contains just the barebones information needed to get +information on failures: what voted -1, what tests failed, what subsystems are long +running (configurable with the `--brief-report-long` opton), and a list of any +attached log files. http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/core.d/builtin-bugsystem.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/builtin-bugsystem.sh b/precommit/core.d/builtin-bugsystem.sh index c7d9a6c..cc14572 100755 --- a/precommit/core.d/builtin-bugsystem.sh +++ b/precommit/core.d/builtin-bugsystem.sh @@ -44,6 +44,7 @@ function console_finalreport declare line declare seccoladj=0 declare spcfx=${PATCH_DIR}/spcl.txt + declare calctime if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then exec 6>&1 @@ -114,6 +115,7 @@ function console_finalreport vote=$(echo "${ourstring}" | cut -f2 -d\|) subs=$(echo "${ourstring}" | cut -f3 -d\|) ela=$(echo "${ourstring}" | cut -f4 -d\|) + calctime=$(clock_display "${ela}") comment=$(echo "${ourstring}" | cut -f5 -d\|) echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}" @@ -121,7 +123,7 @@ function console_finalreport ${SED} -e '1d' "${commentfile1}" > "${commentfile2}" printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \ - "${subs}" "${ela}" "${normaltop}" + "${subs}" "${calctime}" "${normaltop}" while read -r line; do printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}" done < "${commentfile2}" http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.d/briefreport.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/briefreport.sh b/precommit/test-patch.d/briefreport.sh new file mode 100755 index 0000000..144e234 --- /dev/null +++ b/precommit/test-patch.d/briefreport.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_bugsystem brieftext + +BRIEFOUT_LONGRUNNING=3600 + +## @description Usage info for brieftext plugin +## @audience private +## @stability evolving +## @replaceable no +function brieftext_usage +{ + yetus_add_option "--brief-report-file=<file>" "Save a very brief, plain text report to a file" + yetus_add_option "--brief-report-long=<seconds>" "Time in seconds to use as long running subsystem threshold (Default: ${BRIEFOUT_LONGRUNNING})" + +} + +## @description Option parsing for brieftext plugin +## @audience private +## @stability evolving +## @replaceable no +function brieftext_parse_args +{ + declare i + declare fn + + for i in "$@"; do + case ${i} in + --brief-report-file=*) + fn=${i#*=} + ;; + --brief-report-long=*) + BRIEFOUT_LONGRUNNING=${i#*=} + ;; + esac + done + + if [[ -n "${fn}" ]]; then + touch "${fn}" 2>/dev/null + if [[ $? != 0 ]]; then + yetus_error "WARNING: cannot create ${fn}. Ignoring." + else + BRIEFOUT_REPORTFILE=$(yetus_abs "${fn}") + fi + fi +} + +## @description Give access to the brief report file in docker mode +## @audience private +## @stability evolving +## @replaceable no +function brieftext_docker_support +{ + if [[ -n ${BRIEFOUT_REPORTFILE} ]]; then + DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${BRIEFOUT_REPORTFILE}:${BRIEFOUT_REPORTFILE}") + fi +} + +## @description Only print selected information to a report file +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +## @return 0 on success +## @return 1 on failure +function brieftext_finalreport +{ + declare result=$1 + shift + declare i=0 + declare ourstring + declare vote + declare subs + declare ela + declare version + declare -a failed + declare -a long + declare -a filtered + declare hours + declare newtime + declare havelogs=false + + if [[ -z "${BRIEFOUT_REPORTFILE}" ]]; then + return + fi + + big_console_header "Writing Brief Report to ${BRIEFOUT_REPORTFILE}" + + + if [[ ${result} == 0 ]]; then + printf "\n\n+1 overall\n\n" > "${BRIEFOUT_REPORTFILE}" + else + printf "\n\n-1 overall\n\n" > "${BRIEFOUT_REPORTFILE}" + fi + + i=0 + until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + + if [[ ${vote// } = -1 ]]; then + failed=("${failed[@]}" "${subs}") + fi + + if [[ ${vote// } = -0 ]]; then + filtered=("${filtered[@]}" "${subs}") + fi + + if [[ ${ela// } -gt ${BRIEFOUT_LONGRUNNING} ]]; then + long=("${long[@]}" "${subs}") + fi + + ((i=i+1)) + done + + tmparray=($(printf "%s\n" "${failed[@]}" | sort -u)) + failed=("${tmparray[@]}") + tmparray=($(printf "%s\n" "${filtered[@]}" | sort -u)) + filtered=("${tmparray[@]}") + tmparray=($(printf "%s\n" "${long[@]}" | sort -u)) + long=("${tmparray[@]}") + + if [[ ${#failed[@]} -gt 0 ]]; then + { + echo "" + echo "The following subsystems voted -1:" + echo " ${failed[*]}" + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + fi + + if [[ ${#filtered[@]} -gt 0 ]]; then + { + echo "" + echo "The following subsystems voted -1 but" + echo "were configured to be filtered/ignored:" + echo " ${filtered[*]}" + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + fi + + if [[ ${#long[@]} -gt 0 ]]; then + { + echo "" + echo "The following subsystems are considered long running:" + printf "(runtime bigger than " + # We would use clock_display here, but we don't have the + # restrictions that the vote_table has on size plus + # we're almost certainly going to be measured in hours + if [[ ${BRIEFOUT_LONGRUNNING} -ge 3600 ]]; then + hours=$((BRIEFOUT_LONGRUNNING/3600)) + newtime=$((BRIEFOUT_LONGRUNNING-hours*3600)) + printf "%sh %02sm %02ss" ${hours} $((newtime/60)) $((newtime%60)) + else + printf "%sm %02ss" $((BRIEFOUT_LONGRUNNING/60)) $((BRIEFOUT_LONGRUNNING%60)) + fi + echo ")" + echo " ${long[*]}" + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + fi + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { + echo "" + echo "Specific tests:" + } >> "${BRIEFOUT_REPORTFILE}" + + i=0 + until [[ $i -gt ${#TP_TEST_TABLE[@]} ]]; do + ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + { + if [[ -n "${vote// }" ]]; then + echo "" + printf " %s:\n" "${vote}" + echo "" + vote="" + fi + printf " %s\n" "${subs}" + } >> "${BRIEFOUT_REPORTFILE}" + ((i=i+1)) + done + fi + + if [[ -f "${BINDIR}/../VERSION" ]]; then + version=$(cat "${BINDIR}/../VERSION") + elif [[ -f "${BINDIR}/VERSION" ]]; then + version=$(cat "${BINDIR}/VERSION") + fi + + i=0 + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + if [[ "${TP_FOOTER_TABLE[${i}]}" =~ \@\@BASE\@\@ ]]; then + havelogs=true + break + fi + ((i=i+1)) + done + + if [[ "${havelogs}" == true ]]; then + vote="" + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + if [[ "${TP_FOOTER_TABLE[${i}]}" =~ \@\@BASE\@\@ ]]; then + ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') + subs=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f2 -d\|) + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + cut -f3 -d\| | + ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") + # shellcheck disable=SC2016 + size=$(du -sh "${comment// }" | ${AWK} '{print $1}') + if [[ -n "${BUILD_URL}" ]]; then + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + cut -f3 -d\| | + ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") + fi + { + if [[ "${subs}" != "${vote}" ]]; then + echo "" + printf " %s:\n" "${subs// }" + echo "" + vote=${subs} + fi + printf " %s [%s]\n" "${comment}" "${size}" + } >> "${BRIEFOUT_REPORTFILE}" + fi + ((i=i+1)) + done + fi + + { + echo "" + echo "Powered by" "Apache Yetus ${version} http://yetus.apache.org"; + echo "" + } >> "${BRIEFOUT_REPORTFILE}" + +} http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.d/htmlout.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/htmlout.sh b/precommit/test-patch.d/htmlout.sh index bf6d06e..f2dab40 100755 --- a/precommit/test-patch.d/htmlout.sh +++ b/precommit/test-patch.d/htmlout.sh @@ -81,6 +81,7 @@ function htmlout_finalreport declare subs declare color declare comment + declare calctime rm "${commentfile}" 2>/dev/null @@ -98,7 +99,7 @@ function htmlout_finalreport else echo "<tr><th><font color=\"red\">-1 overall</font></th></tr>" fi - echo "</table></tbody>" + echo "</tbody></table>" echo "<p></p>" } > "${commentfile}" @@ -126,6 +127,7 @@ function htmlout_finalreport vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') subs=$(echo "${ourstring}" | cut -f3 -d\|) ela=$(echo "${ourstring}" | cut -f4 -d\|) + calctime=$(clock_display "${ela}") comment=$(echo "${ourstring}" | cut -f5 -d\|) # summary line @@ -147,6 +149,9 @@ function htmlout_finalreport 0) color="blue" ;; + -0) + color="orange" + ;; *) color="black" ;; @@ -157,7 +162,7 @@ function htmlout_finalreport echo "<tr>" printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${vote}" printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}" - printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${ela}" + printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${calctime}" printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${comment}" echo "</tr>" } >> "${commentfile}" http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.d/jira.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/jira.sh b/precommit/test-patch.d/jira.sh index 60afe00..7da9280 100755 --- a/precommit/test-patch.d/jira.sh +++ b/precommit/test-patch.d/jira.sh @@ -357,6 +357,7 @@ function jira_finalreport declare subs declare color declare comment + declare calctime rm "${commentfile}" 2>/dev/null @@ -395,6 +396,7 @@ function jira_finalreport vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') subs=$(echo "${ourstring}" | cut -f3 -d\|) ela=$(echo "${ourstring}" | cut -f4 -d\|) + calctime=$(clock_display "${ela}") comment=$(echo "${ourstring}" | cut -f5 -d\|) # summary line @@ -416,6 +418,9 @@ function jira_finalreport 0) color="blue" ;; + -0) + color="orange" + ;; *) color="black" ;; @@ -425,7 +430,7 @@ function jira_finalreport printf "| {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} |\n" \ "${color}" "${vote}" \ "${color}" "${subs}" \ - "${color}" "${ela}" \ + "${color}" "${calctime}" \ "${color}" "${comment}" \ >> "${commentfile}" ((i=i+1)) http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index ec18134..a818af0 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -220,52 +220,41 @@ function add_header_line ## @description Add to the output table. If the first parameter is a number ## @description that is the vote for that column and calculates the elapsed time -## @description based upon the last start_clock(). If it the string null, then it is -## @description a special entry that signifies extra -## @description content for the final column. The second parameter is the reporting +## @description based upon the last start_clock(). The second parameter is the reporting ## @description subsystem (or test) that is providing the vote. The second parameter ## @description is always required. The third parameter is any extra verbage that goes ## @description with that subsystem. ## @audience public ## @stability stable ## @replaceable no -## @param +1/0/-1/null +## @param +1/0/-1 ## @param subsystem ## @param string -## @return Elapsed time display function add_vote_table { declare value=$1 declare subsystem=$2 shift 2 - declare calctime # apparently shellcheck doesn't know about declare -r #shellcheck disable=SC2155 declare -r elapsed=$(stop_clock) declare filt - yetus_debug "add_vote_table ${value} ${subsystem} ${*}" - - calctime=$(clock_display "${elapsed}") + yetus_debug "add_vote_table ${value} ${subsystem} ${elapsed} ${*}" if [[ ${value} == "1" ]]; then value="+1" fi for filt in "${VOTE_FILTER[@]}"; do - if [[ "${subsystem}" = "${filt}" ]]; then - value=0 + if [[ "${subsystem}" == "${filt}" && "${value}" == -1 ]]; then + value=-0 fi done - if [[ -z ${value} ]]; then - # shellcheck disable=SC2034 - TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | ${subsystem} | | ${*:-} |" - else - # shellcheck disable=SC2034 - TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${calctime} | $* |" - fi + # shellcheck disable=SC2034 + TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${elapsed} | $* |" ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1)) if [[ "${value}" = -1 ]]; then @@ -358,7 +347,7 @@ function finish_vote_table echo "" # shellcheck disable=SC2034 - TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${calctime} | |" + TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${elapsed} | |" ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1 )) } @@ -1863,29 +1852,40 @@ function modules_messages TIMER=${oldtimer} } -## @description Add a test result +## @description Add or update a test result. Update requires +## @description at least the first two parameters. +## @description WARNING: If the message is updated, +## @description then the JDK version is also calculated to match +## @description the current JAVA_HOME. ## @audience public ## @stability evolving ## @replaceable no -## @param module -## @param runtime +## @param moduleindex +## @param -1-0|0|+1 +## @param logvalue +## @param message function module_status { - local index=$1 - local value=$2 - local log=$3 - shift 3 + declare index=$1 + declare value=$2 + shift 2 + declare log=$1 + shift - local jdk + declare jdk jdk=$(report_jvm_version "${JAVA_HOME}") if [[ -n ${index} && ${index} =~ ^[0-9]+$ ]]; then MODULE_STATUS[${index}]="${value}" - MODULE_STATUS_LOG[${index}]="${log}" - MODULE_STATUS_JDK[${index}]=" with JDK v${jdk}" - MODULE_STATUS_MSG[${index}]="${*}" + if [[ -n ${log} ]]; then + MODULE_STATUS_LOG[${index}]="${log}" + fi + if [[ -n $1 ]]; then + MODULE_STATUS_JDK[${index}]=" with JDK v${jdk}" + MODULE_STATUS_MSG[${index}]="${*}" + fi else yetus_error "ASSERT: module_status given bad index: ${index}" yetus_error "ASSERT: module_stats $*" @@ -2097,7 +2097,6 @@ function check_unittests declare statusjdk declare formatresult=0 declare needlog - declare unitlogs if ! verify_needed_test unit; then return 0 @@ -2127,11 +2126,6 @@ function check_unittests ((result=result+$?)) - modules_messages patch unit false - if [[ ${result} == 0 ]]; then - continue - fi - i=0 until [[ $i -eq ${#MODULE[@]} ]]; do module=${MODULE[${i}]} @@ -2155,7 +2149,7 @@ function check_unittests done if [[ ${needlog} == 1 ]]; then - unitlogs="${unitlogs} @@BASE@@/patch-unit-${fn}.txt" + module_status ${i} -1 "patch-unit-${fn}.txt" fi popd >/dev/null @@ -2173,9 +2167,7 @@ function check_unittests done JAVA_HOME=${savejavahome} - if [[ -n "${unitlogs}" ]]; then - add_footer_table "unit test logs" "${unitlogs}" - fi + modules_messages patch unit false if [[ ${JENKINS} == true ]]; then add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"
