http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/findbugs.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/findbugs.sh b/precommit/src/main/shell/test-patch.d/findbugs.sh new file mode 100755 index 0000000..84a807b --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/findbugs.sh @@ -0,0 +1,488 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FINDBUGS_HOME=${FINDBUGS_HOME:-} +FINDBUGS_WARNINGS_FAIL_PRECHECK=false +FINDBUGS_SKIP_MAVEN_SOURCE_CHECK=false + +add_test_type findbugs + +function findbugs_usage +{ + yetus_add_option "--findbugs-home=<path>" "Findbugs home directory (default \${FINDBUGS_HOME})" + yetus_add_option "--findbugs-strict-precheck" "If there are Findbugs warnings during precheck, fail" + yetus_add_option "--findbugs-skip-maven-source-check" "If the buildtool is maven, then skip the source check and run Findbugs for every module" +} + +function findbugs_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} + ;; + --findbugs-strict-precheck) + FINDBUGS_WARNINGS_FAIL_PRECHECK=true + ;; + --findbugs-skip-maven-source-check) + FINDBUGS_SKIP_MAVEN_SOURCE_CHECK=true + ;; + esac + done +} + +## @description initialize the findbugs plug-in +## @audience private +## @stability evolving +## @replaceable no +function findbugs_initialize +{ + if declare -f maven_add_install >/dev/null 2>&1; then + maven_add_install findbugs + fi +} + +function findbugs_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} == maven + || ${BUILDTOOL} == ant ]]; then + if [[ ${filename} =~ \.java$ + || ${filename} =~ (^|/)findbugs-exclude.xml$ ]]; then + add_test findbugs + fi + fi +} + +function findbugs_precheck +{ + declare exec + declare status=0 + + if [[ -z ${FINDBUGS_HOME} ]]; then + yetus_error "FINDBUGS_HOME was not specified." + status=1 + else + for exec in findbugs \ + computeBugHistory \ + convertXmlToText \ + filterBugs \ + setBugDatabaseInfo; do + if ! verify_command "${exec}" "${FINDBUGS_HOME}/bin/${exec}"; then + status=1 + fi + done + fi + if [[ ${status} == 1 ]]; then + add_vote_table 0 findbugs "Findbugs executables are not available." + delete_test findbugs + fi +} + +## @description Dequeue maven modules that lack java sources +## @audience private +## @stability evolving +## @replaceable no +function findbugs_maven_skipper +{ + declare -i i=0 + declare skiplist=() + declare modname + + start_clock + #shellcheck disable=SC2153 + until [[ ${i} -eq ${#MODULE[@]} ]]; do + # If there are no java source code in the module, + # skip parsing output xml file. + if [[ ! -d "${MODULE[${i}]}/src/main/java" ]]; then + skiplist=("${skiplist[@]}" "${MODULE[$i]}") + fi + ((i=i+1)) + done + + i=0 + + for modname in "${skiplist[@]}"; do + dequeue_personality_module "${modname}" + done + + if [[ -n "${modname}" ]]; then + if [[ "${BUILDMODE}" = patch ]]; then + add_vote_table 0 findbugs "Skipped patched modules with no Java source: ${skiplist[*]}" + else + add_vote_table 0 findbugs "Skipped ${#skiplist[@]} modules in the source tree with no Java source." + fi + fi +} + +## @description Run the maven findbugs plugin and record found issues in a bug database +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +## @param repostatus +function findbugs_runner +{ + local name=$1 + local module + local result=0 + local fn + local warnings_file + local i=0 + local savestop + + personality_modules "${name}" findbugs + + # strip out any modules that aren't actually java modules + # this can save a lot of time during testing + if [[ "${BUILDTOOL}" = maven && ${FINDBUGS_SKIP_MAVEN_SOURCE_CHECK} == false ]]; then + findbugs_maven_skipper + fi + + "${BUILDTOOL}_modules_worker" "${name}" findbugs + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + #shellcheck disable=SC2153 + until [[ ${i} -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + start_clock + offset_clock "${MODULE_STATUS_TIMER[${i}]}" + module="${MODULE[${i}]}" + fn=$(module_file_fragment "${module}") + + case ${BUILDTOOL} in + maven) + file="${module}/target/findbugsXml.xml" + ;; + ant) + file="${ANT_FINDBUGSXML}" + ;; + esac + + if [[ ! -f ${file} ]]; then + module_status ${i} -1 "" "${name}/${module} no findbugs output file (${file})" + ((i=i+1)) + continue + fi + + warnings_file="${PATCH_DIR}/${name}-findbugs-${fn}-warnings" + + cp -p "${file}" "${warnings_file}.xml" + + if [[ ${name} == branch ]]; then + "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${PATCH_BRANCH}" \ + "${warnings_file}.xml" "${warnings_file}.xml" + else + "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name patch \ + "${warnings_file}.xml" "${warnings_file}.xml" + fi + if [[ $? != 0 ]]; then + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + module_status ${i} -1 "" "${name}/${module} cannot run setBugDatabaseInfo from findbugs" + ((result=result+1)) + ((i=i+1)) + continue + fi + + "${FINDBUGS_HOME}/bin/convertXmlToText" -html \ + "${warnings_file}.xml" \ + "${warnings_file}.html" + if [[ $? != 0 ]]; then + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + module_status ${i} -1 "" "${name}/${module} cannot run convertXmlToText from findbugs" + ((result=result+1)) + fi + + if [[ -z ${FINDBUGS_VERSION} + && ${name} == branch ]]; then + FINDBUGS_VERSION=$(${GREP} -i "BugCollection version=" "${warnings_file}.xml" \ + | cut -f2 -d\" \ + | cut -f1 -d\" ) + if [[ -n ${FINDBUGS_VERSION} ]]; then + add_footer_table findbugs "v${FINDBUGS_VERSION}" + fi + fi + + ((i=i+1)) + done + return ${result} +} + +## @description Track pre-existing findbugs warnings +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function findbugs_preapply +{ + declare fn + declare module + declare modindex=0 + declare warnings_file + declare module_findbugs_warnings + declare result=0 + declare msg + + if ! verify_needed_test findbugs; then + return 0 + fi + + big_console_header "findbugs detection: ${PATCH_BRANCH}" + + findbugs_runner branch + result=$? + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + until [[ ${modindex} -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${modindex}]} == -1 ]]; then + ((result=result+1)) + ((modindex=modindex+1)) + continue + fi + + module=${MODULE[${modindex}]} + start_clock + offset_clock "${MODULE_STATUS_TIMER[${modindex}]}" + fn=$(module_file_fragment "${module}") + warnings_file="${PATCH_DIR}/branch-findbugs-${fn}-warnings" + # shellcheck disable=SC2016 + module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ + "${PATCH_BRANCH}" \ + "${warnings_file}.xml" \ + "${warnings_file}.xml" \ + | ${AWK} '{print $1}') + + if [[ ${module_findbugs_warnings} -gt 0 ]] ; then + msg="${module} in ${PATCH_BRANCH} has ${module_findbugs_warnings} extant Findbugs warnings." + if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" = "true" ]]; then + module_status ${modindex} -1 "branch-findbugs-${fn}-warnings.html" "${msg}" + ((result=result+1)) + elif [[ "${BUILDMODE}" = full ]]; then + module_status ${modindex} -1 "branch-findbugs-${fn}-warnings.html" "${msg}" + ((result=result+1)) + populate_test_table FindBugs "module:${module}" + #shellcheck disable=SC2162 + while read line; do + firstpart=$(echo "${line}" | cut -f2 -d:) + secondpart=$(echo "${line}" | cut -f9- -d' ') + add_test_table "" "${firstpart}:${secondpart}" + done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${warnings_file}.xml") + else + module_status ${modindex} 0 "branch-findbugs-${fn}-warnings.html" "${msg}" + fi + fi + + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${modindex}]=${savestop} + ((modindex=modindex+1)) + done + modules_messages branch findbugs true + + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description Verify patch does not trigger any findbugs warnings +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function findbugs_postinstall +{ + declare module + declare fn + declare combined_xml + declare branchxml + declare patchxml + declare newbugsbase + declare fixedbugsbase + declare branch_warnings + declare patch_warnings + declare fixed_warnings + declare line + declare firstpart + declare secondpart + declare i=0 + declare result=0 + declare savestop + declare summarize=true + declare statstring + + if ! verify_needed_test findbugs; then + return 0 + fi + + big_console_header "findbugs detection: ${BUILDMODE}" + + findbugs_runner patch + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + + start_clock + offset_clock "${MODULE_STATUS_TIMER[${i}]}" + module="${MODULE[${i}]}" + + buildtool_cwd "${i}" + + fn=$(module_file_fragment "${module}") + + combined_xml="${PATCH_DIR}/combined-findbugs-${fn}.xml" + branchxml="${PATCH_DIR}/branch-findbugs-${fn}-warnings.xml" + patchxml="${PATCH_DIR}/patch-findbugs-${fn}-warnings.xml" + + if [[ -f "${branchxml}" ]]; then + # shellcheck disable=SC2016 + branch_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ + "${PATCH_BRANCH}" \ + "${branchxml}" \ + "${branchxml}" \ + | ${AWK} '{print $1}') + else + branchxml=${patchxml} + fi + + newbugsbase="${PATCH_DIR}/new-findbugs-${fn}" + fixedbugsbase="${PATCH_DIR}/fixed-findbugs-${fn}" + + "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \ + -output "${combined_xml}" \ + "${branchxml}" \ + "${patchxml}" + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run computeBugHistory from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + # shellcheck disable=SC2016 + patch_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ + "patch" \ + "${patchxml}" \ + "${patchxml}" \ + | ${AWK} '{print $1}') + + #shellcheck disable=SC2016 + add_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \ + "${combined_xml}" "${newbugsbase}.xml" | ${AWK} '{print $1}') + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run filterBugs (#1) from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + #shellcheck disable=SC2016 + fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \ + "${combined_xml}" "${fixedbugsbase}.xml" | ${AWK} '{print $1}') + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run filterBugs (#2) from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + statstring=$(generic_calcdiff_status "${branch_warnings}" "${patch_warnings}" "${add_warnings}") + + "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newbugsbase}.xml" \ + "${newbugsbase}.html" + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run convertXmlToText from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + if [[ ${add_warnings} -gt 0 ]] ; then + populate_test_table FindBugs "module:${module}" + #shellcheck disable=SC2162 + while read line; do + firstpart=$(echo "${line}" | cut -f2 -d:) + secondpart=$(echo "${line}" | cut -f9- -d' ') + add_test_table "" "${firstpart}:${secondpart}" + done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newbugsbase}.xml") + + module_status ${i} -1 "new-findbugs-${fn}.html" "${module} ${statstring}" + ((result=result+1)) + elif [[ ${fixed_warnings} -gt 0 ]]; then + module_status ${i} +1 "" "${module} ${statstring}" + summarize=false + fi + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + popd >/dev/null + ((i=i+1)) + done + + modules_messages patch findbugs "${summarize}" + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function findbugs_rebuild +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then + findbugs_preapply + else + findbugs_postinstall + fi +}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/github.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/github.sh b/precommit/src/main/shell/test-patch.d/github.sh new file mode 100755 index 0000000..6b220f1 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/github.sh @@ -0,0 +1,601 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no public APIs here +# SHELLDOC-IGNORE + +# This bug system provides github integration + +add_bugsystem github + +# personalities can override the following settings: + +# Web interface URL. +GITHUB_BASE_URL="https://github.com" + +# API interface URL. +GITHUB_API_URL="https://api.github.com" + +# user/repo +GITHUB_REPO="" + +# user settings +GITHUB_PASSWD="" +GITHUB_TOKEN="" +GITHUB_USER="" +GITHUB_ISSUE="" + +# private globals... +GITHUB_BRIDGED=false +GITHUB_COMMITSHA="" + +# Simple function to set a default GitHub user after PROJECT_NAME has been set +function github_set_github_user +{ + if [[ -n "${PROJECT_NAME}" && ! "${PROJECT_NAME}" = unknown ]]; then + GITHUB_USER=${GITHUB_USER:-"${PROJECT_NAME}qa"} + fi +} + +function github_usage +{ + github_set_github_user + + yetus_add_option "--github-api-url=<url>" "The URL of the API for github (default: '${GITHUB_API_URL}')" + yetus_add_option "--github-base-url=<url>" "The URL of the github server (default:'${GITHUB_BASE_URL}')" + yetus_add_option "--github-password=<pw>" "Github password" + yetus_add_option "--github-repo=<repo>" "github repo to use (default:'${GITHUB_REPO}')" + yetus_add_option "--github-token=<token>" "The token to use to write to github" + yetus_add_option "--github-user=<user>" "Github user [default: ${GITHUB_USER}]" +} + +function github_parse_args +{ + declare i + + github_set_github_user + + for i in "$@"; do + case ${i} in + --github-api-url=*) + GITHUB_API_URL=${i#*=} + ;; + --github-base-url=*) + GITHUB_BASE_URL=${i#*=} + ;; + --github-repo=*) + GITHUB_REPO=${i#*=} + ;; + --github-token=*) + GITHUB_TOKEN=${i#*=} + ;; + --github-password=*) + GITHUB_PASSWD=${i#*=} + ;; + --github-user=*) + GITHUB_USER=${i#*=} + ;; + esac + done +} + + +## @description this gets called when JIRA thinks this +## @description issue is just a pointer to github +## @description WARNING: Called from JIRA plugin! +function github_jira_bridge +{ + declare fileloc=$1 + declare jsonloc=$2 + declare urlfromjira + + # shellcheck disable=SC2016 + urlfromjira=$(${AWK} "match(\$0,\"${GITHUB_BASE_URL}/[^ ]*patch[ &\\\"]\"){url=substr(\$0,RSTART,RLENGTH-1)} + END{if (url) print url}" "${jsonloc}" ) + if [[ -z $urlfromjira ]]; then + # This is currently the expected path, as github pull requests are not common + return 1 + fi + + # we use this to prevent loops later on + GITHUB_BRIDGED=true + yetus_debug "github_jira_bridge: Checking url ${urlfromjira}" + github_breakup_url "${urlfromjira}" + github_locate_patch GH:"${GITHUB_ISSUE}" "${fileloc}" +} + +## @description given a URL, break it up into github plugin globals +## @description this will *override* any personality or yetus defaults +## @param url +function github_breakup_url +{ + declare url=$1 + declare count + declare pos1 + declare pos2 + + count=${url//[^\/]} + count=${#count} + ((pos2=count-3)) + ((pos1=pos2)) + + GITHUB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/) + + ((pos1=pos1+1)) + ((pos2=pos1+1)) + + GITHUB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/) + + ((pos1=pos2+2)) + unset pos2 + + GITHUB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.) +} + + +## @description based upon a github PR, attempt to link back to JIRA +function github_find_jira_title +{ + declare title + declare maybe + declare retval + + if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then + return 1 + fi + + title=$(${GREP} title "${PATCH_DIR}/github-pull.json" \ + | cut -f4 -d\") + + # people typically do two types: JIRA-ISSUE: and [JIRA-ISSUE] + # JIRA_ISSUE_RE is pretty strict so we need to chop that stuff + # out first + + maybe=$(echo "${title}" | cut -f2 -d\[ | cut -f1 -d\]) + jira_determine_issue "${maybe}" + retval=$? + + if [[ ${retval} == 0 ]]; then + return 0 + fi + + maybe=$(echo "${title}" | cut -f1 -d:) + jira_determine_issue "${maybe}" + retval=$? + + if [[ ${retval} == 0 ]]; then + return 0 + fi + + return 1 +} + +function github_determine_issue +{ + declare input=$1 + + if [[ ${input} =~ ^[0-9]+$ + && -n ${GITHUB_REPO} ]]; then + # shellcheck disable=SC2034 + ISSUE=${input} + if [[ -z ${GITHUB_ISSUE} ]]; then + GITHUB_ISSUE=${input} + fi + fi + + # if JIRA didn't call us, should we call it? + if [[ ${GITHUB_BRIDGED} == false ]]; then + github_find_jira_title + if [[ $? == 0 ]]; then + return 0 + fi + fi + + if [[ -n ${GITHUB_ISSUE} ]]; then + return 0 + fi + + return 1 +} + +## @description Try to guess the branch being tested using a variety of heuristics +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success, with PATCH_BRANCH updated appropriately +## @return 1 on failure +function github_determine_branch +{ + if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then + return 1 + fi + + # shellcheck disable=SC2016 + PATCH_BRANCH=$(${AWK} 'match($0,"\"ref\": \""){print $2}' "${PATCH_DIR}/github-pull.json"\ + | cut -f2 -d\"\ + | tail -1 ) + + yetus_debug "Github determine branch: starting with ${PATCH_BRANCH}" + + verify_valid_branch "${PATCH_BRANCH}" +} + +## @description Given input = GH:##, download a patch to output. +## @description Also sets GITHUB_ISSUE to the raw number. +## @audience private +## @stability evolving +## @replaceable no +## @param input +## @param output +## @return 0 on success +## @return 1 on failure +function github_locate_pr_patch +{ + declare input=$1 + declare output=$2 + declare githubauth + + input=${input#GH:} + + # https://github.com/your/repo/pull/## + if [[ ${input} =~ ^${GITHUB_BASE_URL}.*/pull/[0-9]+$ ]]; then + github_breakup_url "${input}.patch" + input=${GITHUB_ISSUE} + fi + + # https://github.com/your/repo/pulls/##.patch + if [[ ${input} =~ ^${GITHUB_BASE_URL}.*patch$ ]]; then + github_breakup_url "${input}" + input=${GITHUB_ISSUE} + fi + + # https://github.com/your/repo/pulls/##.diff + if [[ ${input} =~ ^${GITHUB_BASE_URL}.*diff$ ]]; then + github_breakup_url "${input}" + input=${GITHUB_ISSUE} + fi + + # if it isn't a number at this point, no idea + # how to process + if [[ ! ${input} =~ ^[0-9]+$ ]]; then + yetus_debug "github: ${input} is not a pull request #" + return 1 + fi + + # we always pull the .patch version (even if .diff was given) + # with the assumption that this way binary files work. + # The downside of this is that the patch files are + # significantly larger and therefore take longer to process + PATCHURL="${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}.patch" + echo "GITHUB PR #${input} is being downloaded at $(date) from" + echo "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}" + + if [[ -n "${GITHUB_USER}" + && -n "${GITHUB_PASSWD}" ]]; then + githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" + elif [[ -n "${GITHUB_TOKEN}" ]]; then + githubauth="Authorization: token ${GITHUB_TOKEN}" + else + githubauth="X-ignore-me: fake" + fi + + # Let's pull the PR JSON for later use + ${CURL} --silent --fail \ + -H "Accept: application/vnd.github.v3.full+json" \ + -H "${githubauth}" \ + --output "${PATCH_DIR}/github-pull.json" \ + --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${input}" + + echo "Patch from GITHUB PR #${input} is being downloaded at $(date) from" + echo "${PATCHURL}" + + # the actual patch file + if ! ${CURL} --silent --fail \ + --output "${output}" \ + --location \ + -H "${githubauth}" \ + "${PATCHURL}"; then + yetus_debug "github_locate_patch: not a github pull request." + return 1 + fi + + GITHUB_ISSUE=${input} + + # github will translate this to be #(xx) ! + add_footer_table "GITHUB PR" "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}" + + return 0 +} + + +## @description a wrapper for github_locate_pr_patch that +## @description that takes a (likely checkout'ed) github commit +## @description sha and turns into the the github pr +## @audience private +## @stability evolving +## @replaceable no +## @param input +## @param output +## @return 0 on success +## @return 1 on failure +function github_locate_sha_patch +{ + declare input=$1 + declare output=$2 + declare gitsha + declare number + declare githubauth + + gitsha=${input#GHSHA:} + + # locate the PR number via GitHub API v3 + #curl https://api.github.com/search/issues?q=sha:40a7af3377d8087779bf8ad66397947b7270737a\&type:pr\&repo:apache/yetus + + if [[ -n "${GITHUB_USER}" + && -n "${GITHUB_PASSWD}" ]]; then + githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" + elif [[ -n "${GITHUB_TOKEN}" ]]; then + githubauth="Authorization: token ${GITHUB_TOKEN}" + else + githubauth="X-ignore-me: fake" + fi + + # Let's pull the PR JSON for later use + if ! "${CURL}" --silent --fail \ + -H "Accept: application/vnd.github.v3.full+json" \ + -H "${githubauth}" \ + --output "${PATCH_DIR}/github-search.json" \ + --location \ + "${GITHUB_API_URL}/search/issues?q=${gitsha}&type:pr&repo:${GITHUB_REPO}"; then + return 1 + fi + + # shellcheck disable=SC2016 + number=$("${GREP}" number "${PATCH_DIR}/github-search.json" | \ + head -1 | \ + "${AWK}" '{print $NF}') + number=${number//\s/} + number=${number%,} + + github_locate_pr_patch "GH:${number}" "${output}" + +} + + +## @description Handle the various ways to reference a github PR +## @audience private +## @stability evolving +## @replaceable no +## @param input +## @param output +## @return 0 on success +## @return 1 on failure +function github_locate_patch +{ + declare input=$1 + declare output=$2 + + if [[ "${OFFLINE}" == true ]]; then + yetus_debug "github_locate_patch: offline, skipping" + return 1 + fi + + case "${input}" in + GH:*) + github_locate_pr_patch "${input}" "${output}" + ;; + GHSHA:*) + github_locate_sha_patch "${input}" "${output}" + ;; + esac +} + +function github_linecomments +{ + declare plugin=$1 + declare file=$2 + # shellcheck disable=SC2034 + declare realline=$3 + declare uniline=$4 + declare text=$5 + declare tempfile="${PATCH_DIR}/ghcomment.$$.${RANDOM}" + declare githubauth + + if [[ "${file}" =~ ^./ ]]; then + file=${file##./} + fi + + if [[ -z "${GITHUB_COMMITSHA}" ]]; then + GITHUB_COMMITSHA=$(${GREP} \"sha\" "${PATCH_DIR}/github-pull.json" 2>/dev/null \ + | head -1 \ + | cut -f4 -d\") + fi + + if [[ -z "${uniline}" ]]; then + return + fi + + # build our REST post + { + printf "{\"body\":\"" + echo "${plugin}: ${text}" \ + | ${SED} -e 's,\\,\\\\,g' \ + -e 's,\",\\\",g' \ + -e 's,$,\\r\\n,g' \ + | tr -d '\n' + echo "\"," + echo "\"commit_id\":\"${GITHUB_COMMITSHA}\"," + echo "\"path\":\"${file}\"," + echo "\"position\":${uniline}" + echo "}" + } > "${tempfile}" + + if [[ -n "${GITHUB_USER}" + && -n "${GITHUB_PASSWD}" ]]; then + githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" + elif [[ -n "${GITHUB_TOKEN}" ]]; then + githubauth="Authorization: token ${GITHUB_TOKEN}" + else + return 0 + fi + + ${CURL} -X POST \ + -H "Accept: application/vnd.github.v3.full+json" \ + -H "Content-Type: application/json" \ + -H "${githubauth}" \ + -d @"${tempfile}" \ + --silent --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${GITHUB_ISSUE}/comments" \ + >/dev/null + rm "${tempfile}" +} + +## @description Write the contents of a file to github +## @param filename +## @stability stable +## @audience public +function github_write_comment +{ + declare -r commentfile=${1} + declare retval=0 + declare restfile="${PATCH_DIR}/ghcomment.$$" + declare githubauth + + if [[ "${OFFLINE}" == true ]]; then + echo "Github Plugin: Running in offline, comment skipped." + return 0 + fi + + { + printf "{\"body\":\"" + ${SED} -e 's,\\,\\\\,g' \ + -e 's,\",\\\",g' \ + -e 's,$,\\r\\n,g' "${commentfile}" \ + | tr -d '\n' + echo "\"}" + } > "${restfile}" + + if [[ -n "${GITHUB_USER}" + && -n "${GITHUB_PASSWD}" ]]; then + githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" + elif [[ -n "${GITHUB_TOKEN}" ]]; then + githubauth="Authorization: token ${GITHUB_TOKEN}" + else + echo "Github Plugin: no credentials provided to write a comment." + return 0 + fi + + ${CURL} -X POST \ + -H "Accept: application/vnd.github.v3.full+json" \ + -H "Content-Type: application/json" \ + -H "${githubauth}" \ + -d @"${restfile}" \ + --silent --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/issues/${GITHUB_ISSUE}/comments" \ + >/dev/null + + retval=$? + rm "${restfile}" + return ${retval} +} + +## @description Print out the finished details to the Github PR +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function github_finalreport +{ + declare result=$1 + declare i + declare commentfile=${PATCH_DIR}/gitcommentfile.$$ + declare comment + + rm "${commentfile}" 2>/dev/null + + if [[ ${ROBOT} = "false" + || -z ${GITHUB_ISSUE} ]] ; then + return 0 + fi + + big_console_header "Adding comment to Github" + + if [[ ${result} == 0 ]]; then + echo ":confetti_ball: **+1 overall**" >> "${commentfile}" + else + echo ":broken_heart: **-1 overall**" >> "${commentfile}" + fi + + printf "\n\n\n\n" >> "${commentfile}" + + i=0 + until [[ ${i} -eq ${#TP_HEADER[@]} ]]; do + printf "%s\n\n" "${TP_HEADER[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + + { + printf "\n\n" + echo "| Vote | Subsystem | Runtime | Comment |" + echo "|:----:|----------:|--------:|:--------|" + } >> "${commentfile}" + + i=0 + until [[ ${i} -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') + comment=$(echo "${ourstring}" | cut -f5 -d\|) + + if [[ "${vote}" = "H" ]]; then + echo "||| _${comment}_ |" >> "${commentfile}" + else + echo "${TP_VOTE_TABLE[${i}]}" >> "${commentfile}" + fi + ((i=i+1)) + done + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { + printf "\n\n" + echo "| Reason | Tests |" + echo "|-------:|:------|" + } >> "${commentfile}" + i=0 + until [[ ${i} -eq ${#TP_TEST_TABLE[@]} ]]; do + echo "${TP_TEST_TABLE[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + fi + + { + printf "\n\n" + echo "| Subsystem | Report/Notes |" + echo "|----------:|:-------------|" + } >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") + printf "%s\n" "${comment}" >> "${commentfile}" + ((i=i+1)) + done + + printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" + + github_write_comment "${commentfile}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/gradle.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/gradle.sh b/precommit/src/main/shell/test-patch.d/gradle.sh new file mode 100755 index 0000000..8c78ab4 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/gradle.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# there's nothing in here public, so don't publish docs +# SHELLDOC-IGNORE + +add_build_tool gradle + +declare -a GRADLE_ARGS=() + +function gradle_usage +{ + yetus_add_option "--gradle-cmd=<cmd>" "The 'gradle' command to use (default 'gradle')" + yetus_add_option "--gradlew-cmd=<cmd>" "The 'gradlew' command to use (default 'basedir/gradlew')" +} + +function gradle_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --gradle-cmd=*) + GRADLE=${i#*=} + ;; + --gradlew-cmd=*) + GRADLEW=${i#*=} + ;; + esac + done + + # if we requested offline, pass that to mvn + if [[ ${OFFLINE} == "true" ]]; then + GRADLE_ARGS=("${GRADLE_ARGS[@]}" --offline) + fi + + GRADLE=${GRADLE:-gradle} + GRADLEW=${GRADLEW:-"${BASEDIR}/gradlew"} +} + +function gradle_precheck +{ + declare gradle_version + if ! verify_command gradle "${GRADLE}"; then + add_vote_table -1 gradle "ERROR: gradle is not available." + return 1 + fi + # finally let folks know what version they'll be dealing with. + gradle_version=$(${GRADLE} --version 2>/dev/null | grep Gradle 2>/dev/null) + add_footer_table gradle "version: ${gradle_version}" + return 0 +} + +function gradle_initialize +{ + if [[ "${BUILDTOOL}" = gradle ]]; then + # shellcheck disable=SC2034 + BUILDTOOLCWD=basedir + fi + + # we need to do this before docker kicks in + if [[ -e "${HOME}/.gradle" + && ! -d "${HOME}/.gradle" ]]; then + yetus_error "ERROR: ${HOME}/.gradle is not a directory." + return 1 + elif [[ ! -e "${HOME}/.gradle" ]]; then + yetus_debug "Creating ${HOME}/.gradle" + mkdir -p "${HOME}/.gradle" + fi +} + +function gradle_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ build\.gradle$ + || ${filename} =~ gradlew$ + || ${filename} =~ gradle\.properties$ + || ${filename} =~ wrapper\.gradle$ ]]; then + yetus_debug "tests/compile: ${filename}" + add_test compile + fi +} + +function gradle_buildfile +{ + echo "gradlew" +} + +function gradle_executor +{ + echo "${GRADLEW}" "${GRADLE_ARGS[@]}" +} + +## @description Bootstrap gradle +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function gradle_precompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != gradle ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + big_console_header "gradle boostrap: ${PATCH_BRANCH}" + else + big_console_header "gradle bootstrap: ${BUILDMODE}" + fi + + personality_modules "${repostatus}" gradleboot + + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/${repostatus}-gradle-bootstrap.txt" gradle -b bootstrap.gradle + popd >/dev/null + + modules_workers "${repostatus}" gradleboot + result=$? + modules_messages "${repostatus}" gradleboot true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function gradle_javac_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "\.java" "${input}" > "${output}" +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function gradle_javadoc_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "javadoc.*\.java" "${input}" > "${output}" +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function gradle_scaladoc_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "^\[ant:scaladoc\] /.*\.scala" "${input}" > "${output}" +} + +function gradle_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + shift 2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + checkstyle) + modules_workers "${repostatus}" "${tst}" checkstyleMain checkstyleTest + ;; + compile) + modules_workers "${repostatus}" "${tst}" + ;; + distclean) + modules_workers "${repostatus}" clean + ;; + javadoc) + modules_workers "${repostatus}" "${tst}" javadoc + ;; + scaladoc) + modules_workers "${repostatus}" "${tst}" scaladoc + ;; + unit) + modules_workers "${repostatus}" "${tst}" test + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +function gradle_builtin_personality_modules +{ + local repostatus=$1 + local testtype=$2 + + local module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + for module in "${CHANGED_MODULES[@]}"; do + personality_enqueue_module "${module}" + done +} + +function gradle_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using builtin gradle personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/main/scripts + || ${filename} =~ src/test/scripts + ]]; then + yetus_debug "tests/shell: ${filename}" + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test cc + add_test unit + elif [[ ${filename} =~ \.scala$ ]]; then + add_test scalac + add_test scaladoc + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test javac + add_test javadoc + add_test unit + elif [[ ${filename} =~ src/main ]]; then + yetus_debug "tests/generic+units: ${filename}" + add_test compile + add_test unit + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +function gradle_docker_support +{ + DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${HOME}/.gradle:/home/${USER_NAME}/.gradle") +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/hadolint.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/hadolint.sh b/precommit/src/main/shell/test-patch.d/hadolint.sh new file mode 100755 index 0000000..2392e29 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/hadolint.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no public APIs here +# SHELLDOC-IGNORE + +add_test_type hadolint + +HADOLINT_TIMER=0 +HADOLINT=${HADOLINT:-$(command -v hadolint 2>/dev/null)} + +# files that are going to get hadolint'd +HADOLINT_CHECKFILES=() + +function hadolint_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ Dockerfile$ ]]; then + add_test hadolint + yetus_add_array_element HADOLINT_CHECKFILES "${filename}" + fi +} + +function hadolint_precheck +{ + declare langs + + if ! verify_command "hadolint" "${HADOLINT}"; then + add_vote_table 0 hadolint "hadolint was not available." + delete_test hadolint + fi + + if [[ -z "${LANG}" ]]; then + langs=$(locale -a) + if [[ ${langs} =~ C.UTF-8 ]]; then + yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing C.UTF-8." + export LANG=C.UTF-8 + export LC_ALL=C.UTF-8 + elif [[ ${langs} =~ en_US.UTF-8 ]]; then + yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing en_US.UTF-8." + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + else + for i in ${langs}; do + if [[ "${i}" =~ UTF-8 ]]; then + yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing ${i}." + export LANG="${i}" + export LC_ALL="${i}" + break + fi + done + fi + fi + + if [[ ! "${LANG}" =~ UTF-8 ]]; then + yetus_error "WARNING: hadolint may fail without UTF-8 locale setting." + fi +} + +function hadolint_logic +{ + declare repostatus=$1 + declare i + + pushd "${BASEDIR}" >/dev/null || return 1 + + for i in "${HADOLINT_CHECKFILES[@]}"; do + if [[ -f "${i}" ]]; then + echo " * ${i}" + "${HADOLINT}" "${i}" >> "${PATCH_DIR}/${repostatus}-hadolint-result.txt" + fi + done + popd > /dev/null || return 1 +} + +function hadolint_preapply +{ + if ! verify_needed_test hadolint; then + return 0 + fi + + big_console_header "hadolint plugin: ${PATCH_BRANCH}" + + start_clock + + hadolint_logic branch + + # keep track of how much as elapsed for us already + HADOLINT_TIMER=$(stop_clock) + return 0 +} + +## filename:line\sCODE Text +function hadolint_calcdiffs +{ + declare branch=$1 + declare patch=$2 + declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} + declare j + + # first, pull out just the errors + # shellcheck disable=SC2016 + ${AWK} -F: '{print $NF}' "${branch}" | cut -d' ' -f2- > "${tmp}.branch" + + # shellcheck disable=SC2016 + ${AWK} -F: '{print $NF}' "${patch}" | cut -d' ' -f2- > "${tmp}.patch" + + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${patch}" | tail -1 + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + +function hadolint_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test hadolint; then + return 0 + fi + + big_console_header "hadolint plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${HADOLINT_TIMER}" + + hadolint_logic patch + + calcdiffs \ + "${PATCH_DIR}/branch-hadolint-result.txt" \ + "${PATCH_DIR}/patch-hadolint-result.txt" \ + hadolint \ + > "${PATCH_DIR}/diff-patch-hadolint.txt" + + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/branch-hadolint-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patch-hadolint-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-hadolint.txt" | ${AWK} '{print $1}') + + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_vote_table -1 hadolint "${BUILDMODEMSG} ${statstring}" + add_footer_table hadolint "@@BASE@@/diff-patch-hadolint.txt" + bugsystem_linecomments "hadolint" "${PATCH_DIR}/diff-patch-hadolint.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 hadolint "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 hadolint "There were no new hadolint issues." + return 0 +} + +function hadolint_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + hadolint_preapply + else + hadolint_postapply + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/htmlout.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/htmlout.sh b/precommit/src/main/shell/test-patch.d/htmlout.sh new file mode 100755 index 0000000..f637d1a --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/htmlout.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_bugsystem htmlout + +## @description Usage info for htmlout plugin +## @audience private +## @stability evolving +## @replaceable no +function htmlout_usage +{ + yetus_add_option "--html-report-file=<file>" "Save the final report to an HTML-formated file" +} + +## @description Option parsing for htmlout plugin +## @audience private +## @stability evolving +## @replaceable no +function htmlout_parse_args +{ + declare i + declare fn + + for i in "$@"; do + case ${i} in + --html-report-file=*) + fn=${i#*=} + ;; + esac + done + + if [[ -n "${fn}" ]]; then + if : > "${fn}"; then + HTMLOUT_REPORTFILE_ORIG="${fn}" + HTMLOUT_REPORTFILE=$(yetus_abs "${HTMLOUT_REPORTFILE_ORIG}") + else + yetus_error "WARNING: cannot create HTML report file ${fn}. Ignoring." + fi + fi +} + +## @description Give access to the HTML report file in docker mode +## @audience private +## @stability evolving +## @replaceable no +function htmlout_docker_support +{ + if [[ -n ${HTMLOUT_REPORTFILE} ]]; then + DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${HTMLOUT_REPORTFILE}:/testptch/report.htm") + fi +} + + +## @description Write out an HTML version of the final report to a file +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function htmlout_finalreport +{ + declare result=$1 + declare i + declare commentfile="${HTMLOUT_REPORTFILE}" + declare comment + declare vote + declare ourstring + declare ela + declare subs + declare color + declare comment + declare calctime + + rm "${commentfile}" 2>/dev/null + + if [[ -z "${HTMLOUT_REPORTFILE}" ]]; then + return + fi + + big_console_header "Writing HTML to ${commentfile}" + + { + echo "<table><tbody>" + + if [[ ${result} == 0 ]]; then + echo "<tr><th><font color=\"green\">+1 overall</font></th></tr>" + else + echo "<tr><th><font color=\"red\">-1 overall</font></th></tr>" + fi + echo "</tbody></table>" + echo "<p></p>" + } > "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_HEADER[@]} ]]; do + ourstring=$(echo "${TP_HEADER[${i}]}" | tr -s ' ') + comment=$(echo "${ourstring}" | cut -f2 -d\|) + printf "<tr><td>%s</td></tr>\n" "${comment}" + ((i=i+1)) + done + + { + echo "<table><tbody>" + echo "<tr>" + echo "<th>Vote</th>" + echo "<th>Subsystem</th>" + echo "<th>Runtime</th>" + echo "<th>Comment</th>" + echo "</tr>" + } >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + calctime=$(clock_display "${ela}") + comment=$(echo "${ourstring}" | cut -f5 -d\|) + + if [[ "${vote}" = "H" ]]; then + { + echo "<tr>" + printf "\t\t<td></td>" + printf "<td></td>" + printf "<td></td>" + printf "<td><font color=\"%s\">%s</font></td>\n" "brown" "${comment}" + echo "</tr>" + } >> "${commentfile}" + ((i=i+1)) + continue + fi + + # summary line + if [[ -z ${vote} + && -n ${ela} ]]; then + color="black" + elif [[ -z ${vote} ]]; then + # keep same color + true + else + # new vote line + case ${vote} in + 1|"+1") + color="green" + ;; + -1) + color="red" + ;; + 0) + color="blue" + ;; + -0) + color="orange" + ;; + *) + color="black" + ;; + esac + fi + + { + echo "<tr>" + printf "\t\t<td><font color=\"%s\">%s</font></td>" "${color}" "${vote}" + printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}" + printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${calctime}" + printf "<td><font color=\"%s\">%s</font></td>\n" "${color}" "${comment}" + echo "</tr>" + } >> "${commentfile}" + ((i=i+1)) + done + { + echo "</tbody></table>" + echo "<p></p>" + } >> "${commentfile}" + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { + echo "<table><tbody>" + echo "<tr>" + echo "<th>Reason</th>" + echo "<th>Tests</th>" + echo "</tr>" + } >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do + ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') + subs=$(echo "${ourstring}" | cut -f2 -d\|) + comment=$(echo "${ourstring}" | cut -f3 -d\|) + { + echo "<tr>" + printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}" + printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${comment}" + echo "</tr>" + } >> "${commentfile}" + ((i=i+1)) + done + + { + echo "</tbody></table>" + echo "<p></p>" + } >> "${commentfile}" + fi + + { + echo "<table><tbody>" + echo "<tr>" + echo "<th>Subsystem</th>" + echo "<th>Report/Notes</th>" + echo "</tr>" + } >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + ourstring=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g" | + tr -s ' ') + subs=$(echo "${ourstring}" | cut -f2 -d\|) + comment=$(echo "${ourstring}" | cut -f3 -d\|) + { + echo "<tr>" + printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}" + printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${comment}" + echo "</tr>" + } >> "${commentfile}" + ((i=i+1)) + done + { + echo "</tbody></table>" + echo "<p></p>" + } >> "${commentfile}" + + printf "<p>This message was automatically generated.</p>" >> "${commentfile}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/java.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/java.sh b/precommit/src/main/shell/test-patch.d/java.sh new file mode 100755 index 0000000..8515650 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/java.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type javac +add_test_type javadoc + +yetus_add_entry JDK_TEST_LIST javadoc + +JAVA_INITIALIZED=false + +function initialize_java +{ + local i + local jdkdir + local tmplist + + if [[ ${JAVA_INITIALIZED} == true ]]; then + return + else + JAVA_INITIALIZED=true + fi + + # if we are in pre-docker mode, don't do any of + # this work since it's all going to be wrong anyway + if [[ "${DOCKERSUPPORT}" = "true" ]]; then + return 0 + fi + + if declare -f maven_add_install >/dev/null 2>&1; then + maven_add_install javac + maven_add_install javadoc + fi + + if [[ -z ${JAVA_HOME:-} ]]; then + case ${OSTYPE} in + Darwin) + if [[ -z "${JAVA_HOME}" ]]; then + if [[ -x /usr/libexec/java_home ]]; then + JAVA_HOME="$(/usr/libexec/java_home)" + export JAVA_HOME + else + export JAVA_HOME=/Library/Java/Home + fi + fi + ;; + *) + yetus_error "WARNING: JAVA_HOME not defined. Disabling java tests." + delete_test javac + delete_test javadoc + return 1 + ;; + esac + fi + + JAVA_HOME=$(yetus_abs "${JAVA_HOME}") + + for i in ${JDK_DIR_LIST}; do + if [[ -d "${i}" ]]; then + jdkdir=$(yetus_abs "${i}") + if [[ ${jdkdir} != "${JAVA_HOME}" ]]; then + tmplist="${tmplist} ${jdkdir}" + fi + else + yetus_error "WARNING: Cannot locate JDK directory ${i}: ignoring" + fi + done + + JDK_DIR_LIST="${tmplist} ${JAVA_HOME}" + JDK_DIR_LIST=${JDK_DIR_LIST/ } +} + +function javac_initialize +{ + initialize_java +} + +function javadoc_initialize +{ + initialize_java +} + +## @description Verify that ${JAVA_HOME} is defined +## @audience public +## @stability stable +## @replaceable no +## @return 1 - no JAVA_HOME +## @return 0 - JAVA_HOME defined +function javac_precheck +{ + declare javaversion + declare listofjdks + declare i + + start_clock + + if [[ -z ${JAVA_HOME:-} ]]; then + yetus_error "ERROR: JAVA_HOME is not defined." + add_vote_table -1 pre-patch "JAVA_HOME is not defined." + return 1 + fi + + javaversion=$(report_jvm_version "${JAVA_HOME}") + add_footer_table "Default Java" "${javaversion}" + + if [[ -n ${JDK_DIR_LIST} + && ${JDK_DIR_LIST} != "${JAVA_HOME}" ]]; then + for i in ${JDK_DIR_LIST}; do + javaversion=$(report_jvm_version "${i}") + listofjdks="${listofjdks} ${i}:${javaversion}" + done + add_footer_table "Multi-JDK versions" "${listofjdks}" + fi + return 0 +} + +function javac_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + yetus_debug "tests/javac: ${filename}" + add_test javac + add_test compile + fi +} + +function javadoc_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + yetus_debug "tests/javadoc: ${filename}" + add_test javadoc + fi +} + +## @description +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function javac_compile +{ + declare codebase=$1 + declare multijdkmode=$2 + + if ! verify_needed_test javac; then + return 0 + fi + + if [[ ${codebase} = patch ]]; then + yetus_debug "javac: calling generic_postlog_compare compile javac ${multijdkmode}" + generic_postlog_compare compile javac "${multijdkmode}" + fi +} + +## @description Count and compare the number of JavaDoc warnings pre- and post- patch +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function javadoc_rebuild +{ + declare codebase=$1 + declare multijdkmode + + if verify_multijdk_test javadoc; then + multijdkmode=true + else + multijdkmode=false + fi + + if [[ "${codebase}" = branch ]]; then + generic_pre_handler javadoc "${multijdkmode}" + else + generic_post_handler javadoc javadoc "${multijdkmode}" true + fi +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function javac_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "^.*.java:[0-9]*:" "${input}" > "${output}" +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function javadoc_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "^.*.java:[0-9]*:" "${input}" > "${output}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/jira.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/jira.sh b/precommit/src/main/shell/test-patch.d/jira.sh new file mode 100755 index 0000000..c01587e --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/jira.sh @@ -0,0 +1,526 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no public APIs here +# SHELLDOC-IGNORE + +# this bug system handles JIRA. Personalities +# can override the following variables: + +# base JIRA URL +JIRA_URL=${JIRA_URL:-"https://issues.apache.org/jira"} + +# Issue regex to help identify the project +JIRA_ISSUE_RE='' + +# If the issue status is matched with this pattern, the attached patch is regarded as ready to be applied +JIRA_STATUS_RE='Patch Available' + +add_bugsystem jira + +# Simple function to set a default JIRA user after PROJECT_NAME has been set +function jira_set_jira_user +{ + if [[ -n "${PROJECT_NAME}" && ! "${PROJECT_NAME}" = unknown ]]; then + JIRA_USER=${JIRA_USER:-"${PROJECT_NAME}qa"} + fi +} + +function jira_usage +{ + + jira_set_jira_user + + yetus_add_option "--jira-base-url=<url>" "The URL of the JIRA server (default:'${JIRA_URL}')" + yetus_add_option "--jira-issue-re=<expr>" "Bash regular expression to use when trying to find a jira ref in the patch name (default: '${JIRA_ISSUE_RE}')" + yetus_add_option "--jira-password=<pw>" "The password for accessing JIRA" + yetus_add_option "--jira-status-re=<expr>" "Grep regular expression representing the issue status whose patch is applicable to the codebase (default: '${JIRA_STATUS_RE}')" + yetus_add_option "--jira-user=<user>" "The user to access JIRA command (default: ${JIRA_USER})" +} + +function jira_parse_args +{ + declare i + + jira_set_jira_user + + for i in "$@"; do + case ${i} in + --jira-base-url=*) + JIRA_URL=${i#*=} + ;; + --jira-issue-re=*) + JIRA_ISSUE_RE=${i#*=} + ;; + --jira-password=*) + JIRA_PASSWD=${i#*=} + ;; + --jira-status-re=*) + JIRA_STATUS_RE=${i#*=} + ;; + --jira-user=*) + JIRA_USER=${i#*=} + ;; + esac + done +} + +## @description provides issue determination based upon the URL and more. +## @description WARNING: called from the github plugin! +function jira_determine_issue +{ + declare input=$1 + declare patchnamechunk + declare maybeissue + + if [[ -n ${JIRA_ISSUE} ]]; then + return 0 + fi + + # shellcheck disable=SC2016 + patchnamechunk=$(echo "${input}" | ${AWK} -F/ '{print $NF}') + + maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-) + + if [[ ${maybeissue} =~ ${JIRA_ISSUE_RE} ]]; then + # shellcheck disable=SC2034 + ISSUE=${maybeissue} + JIRA_ISSUE=${maybeissue} + add_footer_table "JIRA Issue" "${JIRA_ISSUE}" + return 0 + fi + + return 1 +} + +function jira_http_fetch +{ + declare input=$1 + declare output=$2 + declare ec + + yetus_debug "jira_http_fetch: ${JIRA_URL}/${input}" + if [[ -n "${JIRA_USER}" + && -n "${JIRA_PASSWD}" ]]; then + ${CURL} --silent --fail \ + --user "${JIRA_USER}:${JIRA_PASSWD}" \ + --output "${output}" \ + --location \ + "${JIRA_URL}/${input}" + else + ${CURL} --silent --fail \ + --output "${output}" \ + --location \ + "${JIRA_URL}/${input}" + fi + ec=$? + case "${ec}" in + "0") + ;; + "1") + yetus_debug "jira_http_fetch: Unsupported protocol. Maybe misspelled jira's url?" + ;; + "3") + yetus_debug "jira_http_fetch: ${JIRA_URL}/${input} url is malformed." + ;; + "6") + yetus_debug "jira_http_fetch: Could not resolve host in URL ${JIRA_URL}." + ;; + "22") + yetus_debug "jira_http_fetch: ${JIRA_URL}/${input} returned 4xx status code. Maybe incorrect username/password?" + ;; + *) + yetus_debug "jira_http_fetch: ${JIRA_URL}/${input} returned $ec error code. See https://ec.haxx.se/usingcurl-returns.html for details." + ;; + esac + return ${ec} +} + +function jira_locate_patch +{ + declare input=$1 + declare fileloc=$2 + declare jsonloc + declare relativeurl + declare retval + declare found=false + + yetus_debug "jira_locate_patch: trying ${JIRA_URL}/browse/${input}" + + if [[ "${OFFLINE}" == true ]]; then + yetus_debug "jira_locate_patch: offline, skipping" + return 1 + fi + + jira_http_fetch "browse/${input}" "${PATCH_DIR}/jira" + if [[ $? != 0 ]]; then + yetus_debug "jira_locate_patch: not a JIRA." + return 1 + fi + + # if github is configured check to see if there is a URL in the text + # that is a github patch file or pull request + if [[ -n "${GITHUB_BASE_URL}" ]]; then + jira_determine_issue "${input}" + # Download information via REST API + jsonloc="${PATCH_DIR}/jira-json" + jira_http_fetch "rest/api/2/issue/${input}" "${jsonloc}" + # Parse the downloaded information to check if the issue is + # just a pointer to GitHub. + github_jira_bridge "${fileloc}" "${jsonloc}" + if [[ $? -eq 0 ]]; then + echo "${input} appears to be a Github PR. Switching Modes." + return 0 + fi + yetus_debug "jira_locate_patch: ${input} seemed like a Github PR, but there was a failure." + fi + + # Not reached if there is a successful github plugin return + if [[ $(${GREP} -c "${JIRA_STATUS_RE}" "${PATCH_DIR}/jira") == 0 ]]; then + if [[ ${ROBOT} == true ]]; then + yetus_error "ERROR: ${input} issue status is not matched with \"${JIRA_STATUS_RE}\"." + cleanup_and_exit 1 + else + yetus_error "WARNING: ${input} issue status is not matched with \"${JIRA_STATUS_RE}\"." + fi + fi + + # See https://jira.atlassian.com/browse/JRA-27637 as why we can't use + # the REST interface here. :( + # the assumption here is that attachment id's are given in an + # ascending order. so bigger # == newer file + #shellcheck disable=SC2016 + tr '>' '\n' < "${PATCH_DIR}/jira" \ + | ${AWK} 'match($0,"/secure/attachment/[0-9]*/[^\"]*"){print substr($0,RSTART,RLENGTH)}' \ + | ${GREP} -v -e 'htm[l]*$' \ + | ${SED} -e 's,[ ]*$,,g' \ + | sort -n -r -k4 -t/ \ + | uniq \ + > "${PATCH_DIR}/jira-attachments.txt" + + echo "${input} patch is being downloaded at $(date) from" + while read -r relativeurl && [[ ${found} = false ]]; do + PATCHURL="${JIRA_URL}${relativeurl}" + + printf " %s -> " "${PATCHURL}" + + jira_http_fetch "${relativeurl}" "${fileloc}" + retval=$? + if [[ ${retval} == 0 ]]; then + found=true + echo "Downloaded" + elif [[ ${retval} == 22 ]]; then + echo "404" + yetus_debug "Presuming the attachment was deleted, trying the next one (see YETUS-298)" + else + echo "Error (curl returned ${retval})" + break + fi + done < <(cat "${PATCH_DIR}/jira-attachments.txt") + + if [[ "${found}" = false ]]; then + yetus_error "ERROR: ${input} could not be downloaded." + cleanup_and_exit 1 + fi + + if [[ ! ${PATCHURL} =~ \.patch$ ]]; then + guess_patch_file "${fileloc}" + if [[ $? == 0 ]]; then + yetus_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. Proceeding, but issue/branch matching might go awry." + add_vote_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${PATCH_NAMING_RULE} for instructions." + else + # this definitely isn't a patch so just bail out. + return 1 + fi + fi + add_footer_table "JIRA Patch URL" "${PATCHURL}" + + return 0 +} + +## @description Try to guess the branch being tested using a variety of heuristics +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success, with PATCH_BRANCH updated appropriately +function jira_determine_branch +{ + declare patchnamechunk + declare total + declare count + declare hinttype + + for hinttype in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do + if [[ -z "${hinttype}" ]]; then + continue + fi + + # If one of these matches the JIRA issue regex + # then we don't want it to trigger the branch + # detection since that's almost certainly not + # intended. In other words, if ISSUE-99 is the + # name of a branch, you want to test ISSUE-99 + # against master, not ISSUE-99's branch + if [[ ${hinttype} =~ ${JIRA_ISSUE_RE} ]]; then + continue + fi + + yetus_debug "Determine branch: starting with ${hinttype}" + patchnamechunk=$(echo "${hinttype}" \ + | ${SED} -e 's,.*/\(.*\)$,\1,' \ + -e 's,\.txt,.,' \ + -e 's,.patch,.,g' \ + -e 's,.diff,.,g' \ + -e 's,\.\.,.,g' \ + -e 's,\.$,,g' ) + + # ISSUE-branch-## + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-) + yetus_debug "Determine branch: ISSUE-branch-## = ${PATCH_BRANCH}" + if [[ -n "${PATCH_BRANCH}" ]]; then + if verify_valid_branch "${PATCH_BRANCH}"; then + return 0 + fi + fi + + # ISSUE-##[.##].branch + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d. ) + count="${PATCH_BRANCH//[^.]}" + total=${#count} + ((total = total + 3 )) + until [[ ${total} -lt 3 ]]; do + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3-${total} -d.) + yetus_debug "Determine branch: ISSUE[.##].branch = ${PATCH_BRANCH}" + ((total=total-1)) + if [[ -n "${PATCH_BRANCH}" ]]; then + if verify_valid_branch "${PATCH_BRANCH}"; then + return 0 + fi + fi + done + + # ISSUE.branch.## + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2- -d. ) + count="${PATCH_BRANCH//[^.]}" + total=${#count} + ((total = total + 3 )) + until [[ ${total} -lt 2 ]]; do + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2-${total} -d.) + yetus_debug "Determine branch: ISSUE.branch[.##] = ${PATCH_BRANCH}" + ((total=total-1)) + if [[ -n "${PATCH_BRANCH}" ]]; then + if verify_valid_branch "${PATCH_BRANCH}"; then + return 0 + fi + fi + done + + # ISSUE-branch.## + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1- -d. ) + count="${PATCH_BRANCH//[^.]}" + total=${#count} + ((total = total + 1 )) + until [[ ${total} -lt 1 ]]; do + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1-${total} -d. ) + yetus_debug "Determine branch: ISSUE-branch[.##] = ${PATCH_BRANCH}" + ((total=total-1)) + if [[ -n "${PATCH_BRANCH}" ]]; then + if verify_valid_branch "${PATCH_BRANCH}"; then + return 0 + fi + fi + done + done + + return 1 +} + +## @description Write the contents of a file to JIRA +## @param filename +## @stability stable +## @audience public +## @return exit code from posting to jira +function jira_write_comment +{ + declare -r commentfile=${1} + declare retval=0 + + if [[ "${OFFLINE}" == true ]]; then + echo "JIRA Plugin: Running in offline, comment skipped." + return 0 + fi + + if [[ -n ${JIRA_PASSWD} + && -n ${JIRA_USER} ]]; then + + # RESTify the comment + { + echo "{\"body\":\"" + ${SED} -e 's,\\,\\\\,g' \ + -e 's,\",\\\",g' \ + -e 's,$,\\r\\n,g' "${commentfile}" \ + | tr -d '\n' + echo "\"}" + } > "${PATCH_DIR}/jiracomment.$$" + + ${CURL} -X POST \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -u "${JIRA_USER}:${JIRA_PASSWD}" \ + -d @"${PATCH_DIR}/jiracomment.$$" \ + --silent --location \ + "${JIRA_URL}/rest/api/2/issue/${JIRA_ISSUE}/comment" \ + >/dev/null + retval=$? + rm "${PATCH_DIR}/jiracomment.$$" + else + echo "JIRA Plugin: no credentials provided to write a comment." + fi + return ${retval} +} + +## @description Print out the finished details to the JIRA issue +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function jira_finalreport +{ + declare result=$1 + declare i + declare commentfile=${PATCH_DIR}/jiracommentfile + declare comment + declare vote + declare ourstring + declare ela + declare subs + declare color + declare comment + declare calctime + + rm "${commentfile}" 2>/dev/null + + if [[ ${ROBOT} == "false" + || ${OFFLINE} == true ]] ; then + return 0 + fi + + if [[ -z "${JIRA_ISSUE}" ]]; then + return 0 + fi + + big_console_header "Adding comment to JIRA" + + if [[ ${result} == 0 ]]; then + echo "| (/) *{color:green}+1 overall{color}* |" >> "${commentfile}" + else + echo "| (x) *{color:red}-1 overall{color}* |" >> "${commentfile}" + fi + + echo "\\\\" >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_HEADER[@]} ]]; do + printf "%s\n" "${TP_HEADER[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + + echo "\\\\" >> "${commentfile}" + + echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + calctime=$(clock_display "${ela}") + comment=$(echo "${ourstring}" | cut -f5 -d\|) + + if [[ "${vote}" = "H" ]]; then + echo "|| || || || {color:brown}${comment}{color} ||" >> "${commentfile}" + ((i=i+1)) + continue + fi + + # summary line + if [[ -z ${vote} + && -n ${ela} ]]; then + color="black" + elif [[ -z ${vote} ]]; then + # keep same color + true + else + # new vote line + case ${vote} in + 1|"+1") + color="green" + ;; + -1) + color="red" + ;; + 0) + color="blue" + ;; + -0) + color="orange" + ;; + H) + # this never gets called (see above) but this is here so others know the color is taken + color="brown" + ;; + *) + color="black" + ;; + esac + fi + + printf "| {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} |\n" \ + "${color}" "${vote}" \ + "${color}" "${subs}" \ + "${color}" "${calctime}" \ + "${color}" "${comment}" \ + >> "${commentfile}" + ((i=i+1)) + done + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + echo "|| Reason || Tests ||" >> "${commentfile}" + i=0 + until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do + printf "%s\n" "${TP_TEST_TABLE[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + fi + + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + echo "|| Subsystem || Report/Notes ||" >> "${commentfile}" + i=0 + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") + printf "%s\n" "${comment}" >> "${commentfile}" + ((i=i+1)) + done + + printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" + + jira_write_comment "${commentfile}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/junit.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/junit.sh b/precommit/src/main/shell/test-patch.d/junit.sh new file mode 100755 index 0000000..0bbe23e --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/junit.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_format junit + +JUNIT_TEST_TIMEOUTS="" +JUNIT_FAILED_TESTS="" + +JUNIT_TEST_OUTPUT_DIR="." +JUNIT_TEST_PREFIX="org.apache." + +function junit_usage +{ + yetus_add_option "--junit-test-output=<path>" "Directory to search for the test output TEST-*.xml files, relative to the module directory (default:'${JUNIT_TEST_OUTPUT_DIR}')" + yetus_add_option "--junit-test-prefix=<prefix to trim>" "Prefix of test names to be be removed. Used to shorten test names by removing common package name. (default:'${JUNIT_TEST_PREFIX}')" +} + +function junit_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --junit-test-output=*) + JUNIT_TEST_OUTPUT_DIR=${i#*=} + ;; + --junit-test-prefix=*) + JUNIT_TEST_PREFIX=${i#*=} + ;; + esac + done +} + +function junit_process_tests +{ + # shellcheck disable=SC2034 + declare module=$1 + declare buildlogfile=$2 + declare result=0 + declare module_test_timeouts + declare module_failed_tests + + # shellcheck disable=SC2016 + module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${buildlogfile}") + if [[ -n "${module_test_timeouts}" ]] ; then + JUNIT_TEST_TIMEOUTS="${JUNIT_TEST_TIMEOUTS} ${module_test_timeouts}" + ((result=result+1)) + fi + + #shellcheck disable=SC2026,SC2038,SC2016 + module_failed_tests=$(find "${JUNIT_TEST_OUTPUT_DIR}" -name 'TEST*.xml'\ + | xargs "${GREP}" -l -E "<failure|<error"\ + | ${AWK} -F/ '{sub("'"TEST-${JUNIT_TEST_PREFIX}"'",""); sub(".xml",""); print $NF}') + if [[ -n "${module_failed_tests}" ]] ; then + JUNIT_FAILED_TESTS="${JUNIT_FAILED_TESTS} ${module_failed_tests}" + ((result=result+1)) + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +function junit_finalize_results +{ + declare jdk=$1 + + if [[ -n "${JUNIT_FAILED_TESTS}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "${jdk}Failed junit tests" ${JUNIT_FAILED_TESTS} + JUNIT_FAILED_TESTS="" + fi + if [[ -n "${JUNIT_TEST_TIMEOUTS}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "${jdk}Timed out junit tests" ${JUNIT_TEST_TIMEOUTS} + JUNIT_TEST_TIMEOUTS="" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/make.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/make.sh b/precommit/src/main/shell/test-patch.d/make.sh new file mode 100755 index 0000000..6fb2b59 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/make.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRCMAKEIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool make + +MAKE=make +MAKEFILE=Makefile + +## @description make usage hook +## @audience private +## @stability evolving +## @replaceable no +function make_usage +{ + yetus_add_option "--make-cmd=<cmd>" "The 'make' command to use (default: '${MAKE}')" + yetus_add_option "--make-file=<filename>" "The name of the file the make cmd should work on (default: '${MAKEFILE}')" +} + +## @description precheck make +## @audience private +## @stability evolving +## @replaceable no +function make_precheck +{ + declare make_version + if ! verify_command make "${MAKE}"; then + add_vote_table -1 make "make was not available." + return 1 + fi + # finally let folks know what version they'll be dealing with. + if make_version=$(set -o pipefail; ${MAKE} --version 2>/dev/null | head -n 1 2>/dev/null) && [ -n "${make_version}" ]; then + add_footer_table make "version: ${make_version}" + fi + return 0 +} + +## @description make argument parser +## @audience private +## @stability evolving +## @param args +function make_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --make-cmd=*) + MAKE=${i#*=} + ;; + --make-file=*) + MAKEFILE=${i#*=} + ;; + --make-use-git-clean) + MAKE_GITCLEAN=true + ;; + esac + done +} + +## @description get the name of the make build filename +## @audience private +## @stability evolving +## @replaceable no +## @return make build file +function make_buildfile +{ + echo "${MAKEFILE}" +} + +## @description get the name of the make binary +## @audience private +## @stability evolving +## @replaceable no +## @return filename +## @param testname +function make_executor +{ + echo "${MAKE} ${MAKE_ARGS[*]}" +} + +## @description make worker +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +## @param test +function make_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + shift 2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + compile) + modules_workers "${repostatus}" "${tst}" + ;; + distclean) + if [[ ${MAKE_GITCLEAN} = true ]];then + git clean -x -f -d + else + modules_workers "${repostatus}" distclean clean + fi + ;; + unit) + modules_workers "${repostatus}" test test + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +## @description make module queuer +## @audience private +## @stability evolving +## @replaceable no +function make_builtin_personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + + declare module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + for module in "${CHANGED_MODULES[@]}"; do + personality_enqueue_module "${module}" + done +} + +## @description make test determiner +## @audience private +## @stability evolving +## @replaceable no +## @param filename +function make_builtin_personality_file_tests +{ + declare filename=$1 + + yetus_debug "Using builtin make personality_file_tests" + + if [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.hh\.in$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.make$ + || ${filename} =~ Makefile$ + ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test unit + fi +}