Repository: yetus Updated Branches: refs/heads/master e3ea8a463 -> 612a0f2d1
http://git-wip-us.apache.org/repos/asf/yetus/blob/612a0f2d/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index 0260c59..05a075e 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -25,6 +25,8 @@ fi this="${BASH_SOURCE-$0}" BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +BINNAME=${this##*/} +BINNAME=${BINNAME%.sh} STARTINGDIR=$(pwd) USER_PARAMS=("$@") GLOBALTIMER=$(date +"%s") @@ -91,6 +93,9 @@ function setup_defaults CHANGED_UNION_MODULES="" REEXECED=false RESETREPO=false + BUILDMODE=patch + # shellcheck disable=SC2034 + BUILDMODEMSG="The patch" ISSUE="" TIMER=$(date +"%s") BUILDTOOL=maven @@ -231,12 +236,15 @@ function add_header_line ## @return Elapsed time display function add_vote_table { - local value=$1 - local subsystem=$2 + declare value=$1 + declare subsystem=$2 shift 2 - local calctime - local -r elapsed=$(stop_clock) + declare calctime + # apparently shellcheck doesn't know about declare -r + #shellcheck disable=SC2155 + declare -r elapsed=$(stop_clock) + declare filt yetus_debug "add_vote_table ${value} ${subsystem} ${*}" @@ -246,6 +254,12 @@ function add_vote_table value="+1" fi + for filt in "${VOTE_FILTER[@]}"; do + if [[ "${subsystem}" = "${filt}" ]]; then + value=0 + fi + done + if [[ -z ${value} ]]; then # shellcheck disable=SC2034 TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | ${subsystem} | | ${*:-} |" @@ -254,6 +268,10 @@ function add_vote_table TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${calctime} | $* |" fi ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1)) + + if [[ "${value}" = -1 ]]; then + ((RESULT = RESULT + 1)) + fi } ## @description Report the JVM version of the given directory @@ -651,7 +669,6 @@ function relative_dir ## @replaceable no function yetus_usage { - declare bugsys declare jdktlist @@ -665,10 +682,14 @@ function yetus_usage jdktlist=$(echo ${JDK_TEST_LIST}) jdktlist=${jdktlist// /,} - echo "test-patch.sh [OPTIONS] patch" - echo "" - echo "Where:" - echo " patch is a file, URL, or bugsystem-compatible location of the patch file" + if [[ "${BUILDMODE}" = patch ]]; then + echo "${BINNAME} [OPTIONS] patch" + echo "" + echo "Where:" + echo " patch is a file, URL, or bugsystem-compatible location of the patch file" + else + echo "${BINNAME} [OPTIONS]" + fi echo "" echo "Options:" echo "" @@ -682,6 +703,7 @@ function yetus_usage yetus_add_option "--contrib-guide=<url>" "URL to point new users towards project conventions. (default: ${PATCH_NAMING_RULE} )" yetus_add_option "--debug" "If set, then output some extra stuff to stderr" yetus_add_option "--dirty-workspace" "Allow the local git workspace to have uncommitted changes" + yetus_add_option "--empty-patch" "Create a summary of the current source tree" yetus_add_option "--java-home=<path>" "Set JAVA_HOME (In Docker mode, this should be local to the image)" yetus_add_option "--linecomments=<bug>" "Only write line comments to this comma delimited list (defaults to bugcomments)" yetus_add_option "--list-plugins" "List all installed plug-ins and then exit" @@ -700,6 +722,7 @@ function yetus_usage yetus_add_option "--summarize=<bool>" "Allow tests to summarize results" yetus_add_option "--test-parallel=<bool>" "Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)" yetus_add_option "--test-threads=<int>" "Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)" + yetus_add_option "--tests-filter=<list>" "Lists of tests to turn failures into warnings" yetus_add_option "--user-plugins=<dir>" "A directory of user provided plugins. see test-patch.d for examples (default empty)" yetus_add_option "--version" "Print release version information and exit" @@ -725,6 +748,7 @@ function yetus_usage yetus_add_option "--build-url=<url>" "Set the build location web page (Default: '${BUILD_URL}')" yetus_add_option "--build-url-console=<location>" "Location relative to --build-url of the console (Default: '${BUILD_URL_CONSOLE}')" yetus_add_option "--build-url-patchdir=<location>" "Location relative to --build-url of the --patch-dir (Default: '${BUILD_URL_ARTIFACTS}')" + yetus_add_option "--console-report-file=<file>" "Save the final console-based report to a file in addition to the screen" yetus_add_option "--console-urls" "Use the build URL instead of path on the console report" yetus_add_option "--instance=<string>" "Parallel execution identifier string" yetus_add_option "--jenkins" "Enable Jenkins-specifc handling (auto: --robot)" @@ -789,6 +813,9 @@ function parse_args # shellcheck disable=SC2034 BUILD_URL_CONSOLE=${i#*=} ;; + --console-report-file=*) + CONSOLE_REPORT_FILE=${i#*=} + ;; --console-urls) # shellcheck disable=SC2034 CONSOLE_USE_BUILD_URL=true @@ -802,6 +829,11 @@ function parse_args --instance=*) INSTANCE=${i#*=} ;; + --empty-patch) + BUILDMODE=full + # shellcheck disable=SC2034 + BUILDMODEMSG="The source tree" + ;; --java-home=*) JAVA_HOME=${i#*=} ;; @@ -866,6 +898,9 @@ function parse_args # shellcheck disable=SC2034 TEST_THREADS=${i#*=} ;; + --tests-filter=*) + yetus_comma_to_array VOTE_FILTER "${i#*=}" + ;; --tpglobaltimer=*) GLOBALTIMER=${i#*=} ;; @@ -892,7 +927,8 @@ function parse_args docker_parse_args "$@" - if [[ -z "${PATCH_OR_ISSUE}" ]]; then + if [[ -z "${PATCH_OR_ISSUE}" + && "${BUILDMODE}" = patch ]]; then yetus_usage exit 1 fi @@ -940,6 +976,7 @@ function parse_args PATCH_DIR="${USER_PATCH_DIR}" fi + # we need absolute dir for PATCH_DIR cd "${STARTINGDIR}" || cleanup_and_exit 1 if [[ ! -d ${PATCH_DIR} ]]; then mkdir -p "${PATCH_DIR}" @@ -950,10 +987,20 @@ function parse_args cleanup_and_exit 1 fi fi - - # we need absolute dir for PATCH_DIR PATCH_DIR=$(yetus_abs "${PATCH_DIR}") + # we need absolute dir for ${CONSOLE_REPORT_FILE} + if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then + touch "${CONSOLE_REPORT_FILE}" + if [[ $? != 0 ]]; then + yetus_error "ERROR: cannot write to ${CONSOLE_REPORT_FILE}. Disabling console report file." + unset CONSOLE_REPORT_FILE + else + j="${CONSOLE_REPORT_FILE}" + CONSOLE_REPORT_FILE=$(yetus_abs "${j}") + fi + fi + if [[ ${RESETREPO} == "true" ]] ; then yetus_add_entry EXEC_MODES ResetRepo fi @@ -1000,7 +1047,7 @@ function find_buildfile_dir yetus_debug "ERROR: ${buildfile} is not found." return 1 else - dir=$(dirname "${dir}") + dir=$(faster_dirname "${dir}") fi done } @@ -1013,17 +1060,29 @@ function find_buildfile_dir function find_changed_files { declare line + declare oldifs - # get a list of all of the files that have been changed, - # except for /dev/null (which would be present for new files). - # Additionally, remove any a/ b/ patterns at the front of the patch filenames. - # shellcheck disable=SC2016 - while read -r line; do - CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}") - done < <( - ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}} - /^diff --git / { p($3); p($4) } - /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u) + case "${BUILDMODE}" in + full) + echo "Building a list of all files in the source tree" + oldifs=${IFS} + IFS=$'\n' + CHANGED_FILES=($(git ls-files)) + IFS=${oldifs} + ;; + patch) + # get a list of all of the files that have been changed, + # except for /dev/null (which would be present for new files). + # Additionally, remove any a/ b/ patterns at the front of the patch filenames. + # shellcheck disable=SC2016 + while read -r line; do + CHANGED_FILES=("${CHANGED_FILES[@]}" "${line}") + done < <( + ${AWK} 'function p(s){sub("^[ab]/","",s); if(s!~"^/dev/null"){print s}} + /^diff --git / { p($3); p($4) } + /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u) + ;; + esac } ## @description Check for directories to skip during @@ -1056,7 +1115,7 @@ function module_skipdir if [[ ${dir} == "." || ${dir} == "/" ]]; then return 0 else - dir=$(dirname "${dir}") + dir=$(faster_dirname "${dir}") yetus_debug "Trying to skip: ${dir}" fi done @@ -1093,8 +1152,15 @@ function find_changed_modules if [[ -z ${buildfile} ]]; then tmpmods=(".") else + # Now find all the modules that were changed for i in "${CHANGED_FILES[@]}"; do + + # TODO: optimize this + if [[ "${BUILDMODE}" = full && ! "${i}" =~ ${buildfile} ]]; then + continue + fi + dirt=$(dirname "${i}") module_skipdir "${dirt}" @@ -1263,8 +1329,10 @@ function git_checkout currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD) if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then - echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}." - echo "WARNING: Continuing anyway..." + if [[ "${BUILDMODE}" = patch ]]; then + echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}." + echo "WARNING: Continuing anyway..." + fi PATCH_BRANCH=${currentbranch} fi fi @@ -1389,8 +1457,11 @@ function determine_issue ## @replaceable no function determine_needed_tests { - local i - local plugin + declare i + declare plugin + + big_console_header "Determining needed tests" + echo "(Depending upon input size and number of plug-ins, this may take a while)" for i in "${CHANGED_FILES[@]}"; do yetus_debug "Determining needed tests for ${i}" @@ -1562,6 +1633,7 @@ function check_reexec declare tpdir declare copy=false declare testdir + declare plugin if [[ ${REEXECED} == true ]]; then big_console_header "Re-exec mode detected. Continuing." @@ -1583,7 +1655,7 @@ function check_reexec fi done - if [[ ${copy} == true ]]; then + if [[ ${copy} == true && "${BUILDMODE}" != full ]]; then big_console_header "precommit patch detected" if [[ ${RESETREPO} == false ]]; then @@ -1622,9 +1694,11 @@ function check_reexec # if we are doing docker, then we re-exec, but underneath the # container - if declare -f ${BUILDTOOL}_docker_support >/dev/null; then - "${BUILDTOOL}_docker_support" - fi + for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${BUGSYSTEMS} ${TESTTYPES} ${TESTFORMATS}; do + if declare -f ${plugin}_docker_support >/dev/null; then + "${plugin}_docker_support" + fi + done TESTPATCHMODE="${USER_PARAMS[*]}" if [[ -n "${BUILD_URL}" ]]; then @@ -1726,7 +1800,9 @@ function modules_messages declare statusjdk declare multijdkmode=false - if [[ ${repostatus} == branch ]]; then + if [[ "${BUILDMODE}" == full ]]; then + repo="the source" + elif [[ "${repostatus}" == branch ]]; then repo=${PATCH_BRANCH} else repo="the patch" @@ -1843,7 +1919,9 @@ function modules_workers declare result=0 declare argv - if [[ ${repostatus} == branch ]]; then + if [[ "${BUILDMODE}" = full ]]; then + repo="the source" + elif [[ ${repostatus} == branch ]]; then repo=${PATCH_BRANCH} else repo="the patch" @@ -2214,8 +2292,6 @@ function runtests verify_patchdir_still_exists check_unittests - - (( RESULT = RESULT + $? )) fi for plugin in ${TESTTYPES}; do @@ -2225,7 +2301,6 @@ function runtests yetus_debug "Running ${plugin}_tests" #shellcheck disable=SC2086 ${plugin}_tests - (( RESULT = RESULT + $? )) fi done } @@ -2368,12 +2443,16 @@ function generic_calcdiff_status ((samepatch=numpatch-addpatch)) ((fixedpatch=numbranch-numpatch+addpatch)) - printf "generated %i new + %i unchanged - %i fixed = %i total (was %i)" \ - "${addpatch}" \ - "${samepatch}" \ - "${fixedpatch}" \ - "${numpatch}" \ - "${numbranch}" + if [[ "${BUILDMODE}" = full ]]; then + printf "has %i issues." "${addpatch}" + else + printf "generated %i new + %i unchanged - %i fixed = %i total (was %i)" \ + "${addpatch}" \ + "${samepatch}" \ + "${fixedpatch}" \ + "${numpatch}" \ + "${numbranch}" + fi } ## @description Helper routine for plugins to ask projects, etc @@ -2581,7 +2660,7 @@ function generic_post_handler return 0 fi - big_console_header "Patch ${testtype} verification" + big_console_header "${testtype} verification: ${BUILDMODE}" for jdkindex in ${JDK_DIR_LIST}; do if [[ ${multijdkmode} == true ]]; then @@ -2705,9 +2784,9 @@ function compile fi if [[ ${codebase} = "branch" ]]; then - big_console_header "Pre-patch ${PATCH_BRANCH} compilation" + big_console_header "${PATCH_BRANCH} compilation: pre-patch" else - big_console_header "Patch compilation" + big_console_header "${PATCH_BRANCH} compilation: ${BUILDMODE}" fi yetus_debug "Is JVM Required? ${JVM_REQUIRED}" @@ -2880,26 +2959,33 @@ function initialize echo "Modes: ${EXEC_MODES}" - locate_patch + if [[ "${BUILDMODE}" = patch ]]; then + locate_patch - # from here on out, we'll be in ${BASEDIR} for cwd - # plugins need to pushd/popd if they change. - git_checkout + # from here on out, we'll be in ${BASEDIR} for cwd + # plugins need to pushd/popd if they change. + git_checkout + + determine_issue + if [[ "${ISSUE}" == 'Unknown' ]]; then + echo "Testing patch on ${PATCH_BRANCH}." + else + echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}." + fi + + patchfile_dryrun_driver "${PATCH_DIR}/patch" + if [[ $? != 0 ]]; then + ((RESULT = RESULT + 1)) + yetus_error "ERROR: ${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}." + add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase required? Wrong Branch? See ${PATCH_NAMING_RULE} for help." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi - determine_issue - if [[ "${ISSUE}" == 'Unknown' ]]; then - echo "Testing patch on ${PATCH_BRANCH}." else - echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}." - fi - patchfile_dryrun_driver "${PATCH_DIR}/patch" - if [[ $? != 0 ]]; then - ((RESULT = RESULT + 1)) - yetus_error "ERROR: ${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}." - add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase required? Wrong Branch? See ${PATCH_NAMING_RULE} for help." - bugsystem_finalreport 1 - cleanup_and_exit 1 + git_checkout + fi find_changed_files @@ -2966,27 +3052,29 @@ function import_core import_core -initialize "$@" +if [[ "${BINNAME}" =~ qbt ]]; then + initialize --empty-patch "$@" +else + initialize "$@" +fi prechecks -patchfiletests -((RESULT=RESULT+$?)) +if [[ "${BUILDMODE}" = patch ]]; then + patchfiletests -compile_cycle branch -((RESULT=RESULT+$?)) + compile_cycle branch -distclean + distclean -apply_patch_file + apply_patch_file -compute_gitdiff + compute_gitdiff +fi compile_cycle patch -((RESULT=RESULT+$?)) runtests -((RESULT=RESULT+$?)) finish_vote_table
