http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/findbugs.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/findbugs.sh b/precommit/test-patch.d/findbugs.sh deleted file mode 100755 index 84a807b..0000000 --- a/precommit/test-patch.d/findbugs.sh +++ /dev/null @@ -1,488 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -FINDBUGS_HOME=${FINDBUGS_HOME:-} -FINDBUGS_WARNINGS_FAIL_PRECHECK=false -FINDBUGS_SKIP_MAVEN_SOURCE_CHECK=false - -add_test_type findbugs - -function findbugs_usage -{ - yetus_add_option "--findbugs-home=<path>" "Findbugs home directory (default \${FINDBUGS_HOME})" - yetus_add_option "--findbugs-strict-precheck" "If there are Findbugs warnings during precheck, fail" - yetus_add_option "--findbugs-skip-maven-source-check" "If the buildtool is maven, then skip the source check and run Findbugs for every module" -} - -function findbugs_parse_args -{ - local i - - for i in "$@"; do - case ${i} in - --findbugs-home=*) - FINDBUGS_HOME=${i#*=} - ;; - --findbugs-strict-precheck) - FINDBUGS_WARNINGS_FAIL_PRECHECK=true - ;; - --findbugs-skip-maven-source-check) - FINDBUGS_SKIP_MAVEN_SOURCE_CHECK=true - ;; - esac - done -} - -## @description initialize the findbugs plug-in -## @audience private -## @stability evolving -## @replaceable no -function findbugs_initialize -{ - if declare -f maven_add_install >/dev/null 2>&1; then - maven_add_install findbugs - fi -} - -function findbugs_filefilter -{ - local filename=$1 - - if [[ ${BUILDTOOL} == maven - || ${BUILDTOOL} == ant ]]; then - if [[ ${filename} =~ \.java$ - || ${filename} =~ (^|/)findbugs-exclude.xml$ ]]; then - add_test findbugs - fi - fi -} - -function findbugs_precheck -{ - declare exec - declare status=0 - - if [[ -z ${FINDBUGS_HOME} ]]; then - yetus_error "FINDBUGS_HOME was not specified." - status=1 - else - for exec in findbugs \ - computeBugHistory \ - convertXmlToText \ - filterBugs \ - setBugDatabaseInfo; do - if ! verify_command "${exec}" "${FINDBUGS_HOME}/bin/${exec}"; then - status=1 - fi - done - fi - if [[ ${status} == 1 ]]; then - add_vote_table 0 findbugs "Findbugs executables are not available." - delete_test findbugs - fi -} - -## @description Dequeue maven modules that lack java sources -## @audience private -## @stability evolving -## @replaceable no -function findbugs_maven_skipper -{ - declare -i i=0 - declare skiplist=() - declare modname - - start_clock - #shellcheck disable=SC2153 - until [[ ${i} -eq ${#MODULE[@]} ]]; do - # If there are no java source code in the module, - # skip parsing output xml file. - if [[ ! -d "${MODULE[${i}]}/src/main/java" ]]; then - skiplist=("${skiplist[@]}" "${MODULE[$i]}") - fi - ((i=i+1)) - done - - i=0 - - for modname in "${skiplist[@]}"; do - dequeue_personality_module "${modname}" - done - - if [[ -n "${modname}" ]]; then - if [[ "${BUILDMODE}" = patch ]]; then - add_vote_table 0 findbugs "Skipped patched modules with no Java source: ${skiplist[*]}" - else - add_vote_table 0 findbugs "Skipped ${#skiplist[@]} modules in the source tree with no Java source." - fi - fi -} - -## @description Run the maven findbugs plugin and record found issues in a bug database -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -## @param repostatus -function findbugs_runner -{ - local name=$1 - local module - local result=0 - local fn - local warnings_file - local i=0 - local savestop - - personality_modules "${name}" findbugs - - # strip out any modules that aren't actually java modules - # this can save a lot of time during testing - if [[ "${BUILDTOOL}" = maven && ${FINDBUGS_SKIP_MAVEN_SOURCE_CHECK} == false ]]; then - findbugs_maven_skipper - fi - - "${BUILDTOOL}_modules_worker" "${name}" findbugs - - if [[ ${UNSUPPORTED_TEST} = true ]]; then - return 0 - fi - - #shellcheck disable=SC2153 - until [[ ${i} -eq ${#MODULE[@]} ]]; do - if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then - ((result=result+1)) - ((i=i+1)) - continue - fi - start_clock - offset_clock "${MODULE_STATUS_TIMER[${i}]}" - module="${MODULE[${i}]}" - fn=$(module_file_fragment "${module}") - - case ${BUILDTOOL} in - maven) - file="${module}/target/findbugsXml.xml" - ;; - ant) - file="${ANT_FINDBUGSXML}" - ;; - esac - - if [[ ! -f ${file} ]]; then - module_status ${i} -1 "" "${name}/${module} no findbugs output file (${file})" - ((i=i+1)) - continue - fi - - warnings_file="${PATCH_DIR}/${name}-findbugs-${fn}-warnings" - - cp -p "${file}" "${warnings_file}.xml" - - if [[ ${name} == branch ]]; then - "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${PATCH_BRANCH}" \ - "${warnings_file}.xml" "${warnings_file}.xml" - else - "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name patch \ - "${warnings_file}.xml" "${warnings_file}.xml" - fi - if [[ $? != 0 ]]; then - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${i}]=${savestop} - module_status ${i} -1 "" "${name}/${module} cannot run setBugDatabaseInfo from findbugs" - ((result=result+1)) - ((i=i+1)) - continue - fi - - "${FINDBUGS_HOME}/bin/convertXmlToText" -html \ - "${warnings_file}.xml" \ - "${warnings_file}.html" - if [[ $? != 0 ]]; then - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${i}]=${savestop} - module_status ${i} -1 "" "${name}/${module} cannot run convertXmlToText from findbugs" - ((result=result+1)) - fi - - if [[ -z ${FINDBUGS_VERSION} - && ${name} == branch ]]; then - FINDBUGS_VERSION=$(${GREP} -i "BugCollection version=" "${warnings_file}.xml" \ - | cut -f2 -d\" \ - | cut -f1 -d\" ) - if [[ -n ${FINDBUGS_VERSION} ]]; then - add_footer_table findbugs "v${FINDBUGS_VERSION}" - fi - fi - - ((i=i+1)) - done - return ${result} -} - -## @description Track pre-existing findbugs warnings -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -function findbugs_preapply -{ - declare fn - declare module - declare modindex=0 - declare warnings_file - declare module_findbugs_warnings - declare result=0 - declare msg - - if ! verify_needed_test findbugs; then - return 0 - fi - - big_console_header "findbugs detection: ${PATCH_BRANCH}" - - findbugs_runner branch - result=$? - - if [[ ${UNSUPPORTED_TEST} = true ]]; then - return 0 - fi - - until [[ ${modindex} -eq ${#MODULE[@]} ]]; do - if [[ ${MODULE_STATUS[${modindex}]} == -1 ]]; then - ((result=result+1)) - ((modindex=modindex+1)) - continue - fi - - module=${MODULE[${modindex}]} - start_clock - offset_clock "${MODULE_STATUS_TIMER[${modindex}]}" - fn=$(module_file_fragment "${module}") - warnings_file="${PATCH_DIR}/branch-findbugs-${fn}-warnings" - # shellcheck disable=SC2016 - module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ - "${PATCH_BRANCH}" \ - "${warnings_file}.xml" \ - "${warnings_file}.xml" \ - | ${AWK} '{print $1}') - - if [[ ${module_findbugs_warnings} -gt 0 ]] ; then - msg="${module} in ${PATCH_BRANCH} has ${module_findbugs_warnings} extant Findbugs warnings." - if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" = "true" ]]; then - module_status ${modindex} -1 "branch-findbugs-${fn}-warnings.html" "${msg}" - ((result=result+1)) - elif [[ "${BUILDMODE}" = full ]]; then - module_status ${modindex} -1 "branch-findbugs-${fn}-warnings.html" "${msg}" - ((result=result+1)) - populate_test_table FindBugs "module:${module}" - #shellcheck disable=SC2162 - while read line; do - firstpart=$(echo "${line}" | cut -f2 -d:) - secondpart=$(echo "${line}" | cut -f9- -d' ') - add_test_table "" "${firstpart}:${secondpart}" - done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${warnings_file}.xml") - else - module_status ${modindex} 0 "branch-findbugs-${fn}-warnings.html" "${msg}" - fi - fi - - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${modindex}]=${savestop} - ((modindex=modindex+1)) - done - modules_messages branch findbugs true - - if [[ ${result} != 0 ]]; then - return 1 - fi - return 0 -} - -## @description Verify patch does not trigger any findbugs warnings -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -function findbugs_postinstall -{ - declare module - declare fn - declare combined_xml - declare branchxml - declare patchxml - declare newbugsbase - declare fixedbugsbase - declare branch_warnings - declare patch_warnings - declare fixed_warnings - declare line - declare firstpart - declare secondpart - declare i=0 - declare result=0 - declare savestop - declare summarize=true - declare statstring - - if ! verify_needed_test findbugs; then - return 0 - fi - - big_console_header "findbugs detection: ${BUILDMODE}" - - findbugs_runner patch - - if [[ ${UNSUPPORTED_TEST} = true ]]; then - return 0 - fi - - until [[ $i -eq ${#MODULE[@]} ]]; do - if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then - ((result=result+1)) - ((i=i+1)) - continue - fi - - start_clock - offset_clock "${MODULE_STATUS_TIMER[${i}]}" - module="${MODULE[${i}]}" - - buildtool_cwd "${i}" - - fn=$(module_file_fragment "${module}") - - combined_xml="${PATCH_DIR}/combined-findbugs-${fn}.xml" - branchxml="${PATCH_DIR}/branch-findbugs-${fn}-warnings.xml" - patchxml="${PATCH_DIR}/patch-findbugs-${fn}-warnings.xml" - - if [[ -f "${branchxml}" ]]; then - # shellcheck disable=SC2016 - branch_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ - "${PATCH_BRANCH}" \ - "${branchxml}" \ - "${branchxml}" \ - | ${AWK} '{print $1}') - else - branchxml=${patchxml} - fi - - newbugsbase="${PATCH_DIR}/new-findbugs-${fn}" - fixedbugsbase="${PATCH_DIR}/fixed-findbugs-${fn}" - - "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \ - -output "${combined_xml}" \ - "${branchxml}" \ - "${patchxml}" - if [[ $? != 0 ]]; then - popd >/dev/null - module_status ${i} -1 "" "${module} cannot run computeBugHistory from findbugs" - ((result=result+1)) - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${i}]=${savestop} - ((i=i+1)) - continue - fi - - # shellcheck disable=SC2016 - patch_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ - "patch" \ - "${patchxml}" \ - "${patchxml}" \ - | ${AWK} '{print $1}') - - #shellcheck disable=SC2016 - add_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \ - "${combined_xml}" "${newbugsbase}.xml" | ${AWK} '{print $1}') - if [[ $? != 0 ]]; then - popd >/dev/null - module_status ${i} -1 "" "${module} cannot run filterBugs (#1) from findbugs" - ((result=result+1)) - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${i}]=${savestop} - ((i=i+1)) - continue - fi - - #shellcheck disable=SC2016 - fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \ - "${combined_xml}" "${fixedbugsbase}.xml" | ${AWK} '{print $1}') - if [[ $? != 0 ]]; then - popd >/dev/null - module_status ${i} -1 "" "${module} cannot run filterBugs (#2) from findbugs" - ((result=result+1)) - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${i}]=${savestop} - ((i=i+1)) - continue - fi - - statstring=$(generic_calcdiff_status "${branch_warnings}" "${patch_warnings}" "${add_warnings}") - - "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newbugsbase}.xml" \ - "${newbugsbase}.html" - if [[ $? != 0 ]]; then - popd >/dev/null - module_status ${i} -1 "" "${module} cannot run convertXmlToText from findbugs" - ((result=result+1)) - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${i}]=${savestop} - ((i=i+1)) - continue - fi - - if [[ ${add_warnings} -gt 0 ]] ; then - populate_test_table FindBugs "module:${module}" - #shellcheck disable=SC2162 - while read line; do - firstpart=$(echo "${line}" | cut -f2 -d:) - secondpart=$(echo "${line}" | cut -f9- -d' ') - add_test_table "" "${firstpart}:${secondpart}" - done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newbugsbase}.xml") - - module_status ${i} -1 "new-findbugs-${fn}.html" "${module} ${statstring}" - ((result=result+1)) - elif [[ ${fixed_warnings} -gt 0 ]]; then - module_status ${i} +1 "" "${module} ${statstring}" - summarize=false - fi - savestop=$(stop_clock) - MODULE_STATUS_TIMER[${i}]=${savestop} - popd >/dev/null - ((i=i+1)) - done - - modules_messages patch findbugs "${summarize}" - if [[ ${result} != 0 ]]; then - return 1 - fi - return 0 -} - -function findbugs_rebuild -{ - declare repostatus=$1 - - if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then - findbugs_preapply - else - findbugs_postinstall - fi -}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/github.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/github.sh b/precommit/test-patch.d/github.sh deleted file mode 100755 index 6b220f1..0000000 --- a/precommit/test-patch.d/github.sh +++ /dev/null @@ -1,601 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# no public APIs here -# SHELLDOC-IGNORE - -# This bug system provides github integration - -add_bugsystem github - -# personalities can override the following settings: - -# Web interface URL. -GITHUB_BASE_URL="https://github.com"; - -# API interface URL. -GITHUB_API_URL="https://api.github.com"; - -# user/repo -GITHUB_REPO="" - -# user settings -GITHUB_PASSWD="" -GITHUB_TOKEN="" -GITHUB_USER="" -GITHUB_ISSUE="" - -# private globals... -GITHUB_BRIDGED=false -GITHUB_COMMITSHA="" - -# Simple function to set a default GitHub user after PROJECT_NAME has been set -function github_set_github_user -{ - if [[ -n "${PROJECT_NAME}" && ! "${PROJECT_NAME}" = unknown ]]; then - GITHUB_USER=${GITHUB_USER:-"${PROJECT_NAME}qa"} - fi -} - -function github_usage -{ - github_set_github_user - - yetus_add_option "--github-api-url=<url>" "The URL of the API for github (default: '${GITHUB_API_URL}')" - yetus_add_option "--github-base-url=<url>" "The URL of the github server (default:'${GITHUB_BASE_URL}')" - yetus_add_option "--github-password=<pw>" "Github password" - yetus_add_option "--github-repo=<repo>" "github repo to use (default:'${GITHUB_REPO}')" - yetus_add_option "--github-token=<token>" "The token to use to write to github" - yetus_add_option "--github-user=<user>" "Github user [default: ${GITHUB_USER}]" -} - -function github_parse_args -{ - declare i - - github_set_github_user - - for i in "$@"; do - case ${i} in - --github-api-url=*) - GITHUB_API_URL=${i#*=} - ;; - --github-base-url=*) - GITHUB_BASE_URL=${i#*=} - ;; - --github-repo=*) - GITHUB_REPO=${i#*=} - ;; - --github-token=*) - GITHUB_TOKEN=${i#*=} - ;; - --github-password=*) - GITHUB_PASSWD=${i#*=} - ;; - --github-user=*) - GITHUB_USER=${i#*=} - ;; - esac - done -} - - -## @description this gets called when JIRA thinks this -## @description issue is just a pointer to github -## @description WARNING: Called from JIRA plugin! -function github_jira_bridge -{ - declare fileloc=$1 - declare jsonloc=$2 - declare urlfromjira - - # shellcheck disable=SC2016 - urlfromjira=$(${AWK} "match(\$0,\"${GITHUB_BASE_URL}/[^ ]*patch[ &\\\"]\"){url=substr(\$0,RSTART,RLENGTH-1)} - END{if (url) print url}" "${jsonloc}" ) - if [[ -z $urlfromjira ]]; then - # This is currently the expected path, as github pull requests are not common - return 1 - fi - - # we use this to prevent loops later on - GITHUB_BRIDGED=true - yetus_debug "github_jira_bridge: Checking url ${urlfromjira}" - github_breakup_url "${urlfromjira}" - github_locate_patch GH:"${GITHUB_ISSUE}" "${fileloc}" -} - -## @description given a URL, break it up into github plugin globals -## @description this will *override* any personality or yetus defaults -## @param url -function github_breakup_url -{ - declare url=$1 - declare count - declare pos1 - declare pos2 - - count=${url//[^\/]} - count=${#count} - ((pos2=count-3)) - ((pos1=pos2)) - - GITHUB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/) - - ((pos1=pos1+1)) - ((pos2=pos1+1)) - - GITHUB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/) - - ((pos1=pos2+2)) - unset pos2 - - GITHUB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.) -} - - -## @description based upon a github PR, attempt to link back to JIRA -function github_find_jira_title -{ - declare title - declare maybe - declare retval - - if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then - return 1 - fi - - title=$(${GREP} title "${PATCH_DIR}/github-pull.json" \ - | cut -f4 -d\") - - # people typically do two types: JIRA-ISSUE: and [JIRA-ISSUE] - # JIRA_ISSUE_RE is pretty strict so we need to chop that stuff - # out first - - maybe=$(echo "${title}" | cut -f2 -d\[ | cut -f1 -d\]) - jira_determine_issue "${maybe}" - retval=$? - - if [[ ${retval} == 0 ]]; then - return 0 - fi - - maybe=$(echo "${title}" | cut -f1 -d:) - jira_determine_issue "${maybe}" - retval=$? - - if [[ ${retval} == 0 ]]; then - return 0 - fi - - return 1 -} - -function github_determine_issue -{ - declare input=$1 - - if [[ ${input} =~ ^[0-9]+$ - && -n ${GITHUB_REPO} ]]; then - # shellcheck disable=SC2034 - ISSUE=${input} - if [[ -z ${GITHUB_ISSUE} ]]; then - GITHUB_ISSUE=${input} - fi - fi - - # if JIRA didn't call us, should we call it? - if [[ ${GITHUB_BRIDGED} == false ]]; then - github_find_jira_title - if [[ $? == 0 ]]; then - return 0 - fi - fi - - if [[ -n ${GITHUB_ISSUE} ]]; then - return 0 - fi - - return 1 -} - -## @description Try to guess the branch being tested using a variety of heuristics -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success, with PATCH_BRANCH updated appropriately -## @return 1 on failure -function github_determine_branch -{ - if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then - return 1 - fi - - # shellcheck disable=SC2016 - PATCH_BRANCH=$(${AWK} 'match($0,"\"ref\": \""){print $2}' "${PATCH_DIR}/github-pull.json"\ - | cut -f2 -d\"\ - | tail -1 ) - - yetus_debug "Github determine branch: starting with ${PATCH_BRANCH}" - - verify_valid_branch "${PATCH_BRANCH}" -} - -## @description Given input = GH:##, download a patch to output. -## @description Also sets GITHUB_ISSUE to the raw number. -## @audience private -## @stability evolving -## @replaceable no -## @param input -## @param output -## @return 0 on success -## @return 1 on failure -function github_locate_pr_patch -{ - declare input=$1 - declare output=$2 - declare githubauth - - input=${input#GH:} - - # https://github.com/your/repo/pull/## - if [[ ${input} =~ ^${GITHUB_BASE_URL}.*/pull/[0-9]+$ ]]; then - github_breakup_url "${input}.patch" - input=${GITHUB_ISSUE} - fi - - # https://github.com/your/repo/pulls/##.patch - if [[ ${input} =~ ^${GITHUB_BASE_URL}.*patch$ ]]; then - github_breakup_url "${input}" - input=${GITHUB_ISSUE} - fi - - # https://github.com/your/repo/pulls/##.diff - if [[ ${input} =~ ^${GITHUB_BASE_URL}.*diff$ ]]; then - github_breakup_url "${input}" - input=${GITHUB_ISSUE} - fi - - # if it isn't a number at this point, no idea - # how to process - if [[ ! ${input} =~ ^[0-9]+$ ]]; then - yetus_debug "github: ${input} is not a pull request #" - return 1 - fi - - # we always pull the .patch version (even if .diff was given) - # with the assumption that this way binary files work. - # The downside of this is that the patch files are - # significantly larger and therefore take longer to process - PATCHURL="${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}.patch" - echo "GITHUB PR #${input} is being downloaded at $(date) from" - echo "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}" - - if [[ -n "${GITHUB_USER}" - && -n "${GITHUB_PASSWD}" ]]; then - githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" - elif [[ -n "${GITHUB_TOKEN}" ]]; then - githubauth="Authorization: token ${GITHUB_TOKEN}" - else - githubauth="X-ignore-me: fake" - fi - - # Let's pull the PR JSON for later use - ${CURL} --silent --fail \ - -H "Accept: application/vnd.github.v3.full+json" \ - -H "${githubauth}" \ - --output "${PATCH_DIR}/github-pull.json" \ - --location \ - "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${input}" - - echo "Patch from GITHUB PR #${input} is being downloaded at $(date) from" - echo "${PATCHURL}" - - # the actual patch file - if ! ${CURL} --silent --fail \ - --output "${output}" \ - --location \ - -H "${githubauth}" \ - "${PATCHURL}"; then - yetus_debug "github_locate_patch: not a github pull request." - return 1 - fi - - GITHUB_ISSUE=${input} - - # github will translate this to be #(xx) ! - add_footer_table "GITHUB PR" "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}" - - return 0 -} - - -## @description a wrapper for github_locate_pr_patch that -## @description that takes a (likely checkout'ed) github commit -## @description sha and turns into the the github pr -## @audience private -## @stability evolving -## @replaceable no -## @param input -## @param output -## @return 0 on success -## @return 1 on failure -function github_locate_sha_patch -{ - declare input=$1 - declare output=$2 - declare gitsha - declare number - declare githubauth - - gitsha=${input#GHSHA:} - - # locate the PR number via GitHub API v3 - #curl https://api.github.com/search/issues?q=sha:40a7af3377d8087779bf8ad66397947b7270737a\&type:pr\&repo:apache/yetus - - if [[ -n "${GITHUB_USER}" - && -n "${GITHUB_PASSWD}" ]]; then - githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" - elif [[ -n "${GITHUB_TOKEN}" ]]; then - githubauth="Authorization: token ${GITHUB_TOKEN}" - else - githubauth="X-ignore-me: fake" - fi - - # Let's pull the PR JSON for later use - if ! "${CURL}" --silent --fail \ - -H "Accept: application/vnd.github.v3.full+json" \ - -H "${githubauth}" \ - --output "${PATCH_DIR}/github-search.json" \ - --location \ - "${GITHUB_API_URL}/search/issues?q=${gitsha}&type:pr&repo:${GITHUB_REPO}"; then - return 1 - fi - - # shellcheck disable=SC2016 - number=$("${GREP}" number "${PATCH_DIR}/github-search.json" | \ - head -1 | \ - "${AWK}" '{print $NF}') - number=${number//\s/} - number=${number%,} - - github_locate_pr_patch "GH:${number}" "${output}" - -} - - -## @description Handle the various ways to reference a github PR -## @audience private -## @stability evolving -## @replaceable no -## @param input -## @param output -## @return 0 on success -## @return 1 on failure -function github_locate_patch -{ - declare input=$1 - declare output=$2 - - if [[ "${OFFLINE}" == true ]]; then - yetus_debug "github_locate_patch: offline, skipping" - return 1 - fi - - case "${input}" in - GH:*) - github_locate_pr_patch "${input}" "${output}" - ;; - GHSHA:*) - github_locate_sha_patch "${input}" "${output}" - ;; - esac -} - -function github_linecomments -{ - declare plugin=$1 - declare file=$2 - # shellcheck disable=SC2034 - declare realline=$3 - declare uniline=$4 - declare text=$5 - declare tempfile="${PATCH_DIR}/ghcomment.$$.${RANDOM}" - declare githubauth - - if [[ "${file}" =~ ^./ ]]; then - file=${file##./} - fi - - if [[ -z "${GITHUB_COMMITSHA}" ]]; then - GITHUB_COMMITSHA=$(${GREP} \"sha\" "${PATCH_DIR}/github-pull.json" 2>/dev/null \ - | head -1 \ - | cut -f4 -d\") - fi - - if [[ -z "${uniline}" ]]; then - return - fi - - # build our REST post - { - printf "{\"body\":\"" - echo "${plugin}: ${text}" \ - | ${SED} -e 's,\\,\\\\,g' \ - -e 's,\",\\\",g' \ - -e 's,$,\\r\\n,g' \ - | tr -d '\n' - echo "\"," - echo "\"commit_id\":\"${GITHUB_COMMITSHA}\"," - echo "\"path\":\"${file}\"," - echo "\"position\":${uniline}" - echo "}" - } > "${tempfile}" - - if [[ -n "${GITHUB_USER}" - && -n "${GITHUB_PASSWD}" ]]; then - githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" - elif [[ -n "${GITHUB_TOKEN}" ]]; then - githubauth="Authorization: token ${GITHUB_TOKEN}" - else - return 0 - fi - - ${CURL} -X POST \ - -H "Accept: application/vnd.github.v3.full+json" \ - -H "Content-Type: application/json" \ - -H "${githubauth}" \ - -d @"${tempfile}" \ - --silent --location \ - "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${GITHUB_ISSUE}/comments" \ - >/dev/null - rm "${tempfile}" -} - -## @description Write the contents of a file to github -## @param filename -## @stability stable -## @audience public -function github_write_comment -{ - declare -r commentfile=${1} - declare retval=0 - declare restfile="${PATCH_DIR}/ghcomment.$$" - declare githubauth - - if [[ "${OFFLINE}" == true ]]; then - echo "Github Plugin: Running in offline, comment skipped." - return 0 - fi - - { - printf "{\"body\":\"" - ${SED} -e 's,\\,\\\\,g' \ - -e 's,\",\\\",g' \ - -e 's,$,\\r\\n,g' "${commentfile}" \ - | tr -d '\n' - echo "\"}" - } > "${restfile}" - - if [[ -n "${GITHUB_USER}" - && -n "${GITHUB_PASSWD}" ]]; then - githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" - elif [[ -n "${GITHUB_TOKEN}" ]]; then - githubauth="Authorization: token ${GITHUB_TOKEN}" - else - echo "Github Plugin: no credentials provided to write a comment." - return 0 - fi - - ${CURL} -X POST \ - -H "Accept: application/vnd.github.v3.full+json" \ - -H "Content-Type: application/json" \ - -H "${githubauth}" \ - -d @"${restfile}" \ - --silent --location \ - "${GITHUB_API_URL}/repos/${GITHUB_REPO}/issues/${GITHUB_ISSUE}/comments" \ - >/dev/null - - retval=$? - rm "${restfile}" - return ${retval} -} - -## @description Print out the finished details to the Github PR -## @audience private -## @stability evolving -## @replaceable no -## @param runresult -function github_finalreport -{ - declare result=$1 - declare i - declare commentfile=${PATCH_DIR}/gitcommentfile.$$ - declare comment - - rm "${commentfile}" 2>/dev/null - - if [[ ${ROBOT} = "false" - || -z ${GITHUB_ISSUE} ]] ; then - return 0 - fi - - big_console_header "Adding comment to Github" - - if [[ ${result} == 0 ]]; then - echo ":confetti_ball: **+1 overall**" >> "${commentfile}" - else - echo ":broken_heart: **-1 overall**" >> "${commentfile}" - fi - - printf "\n\n\n\n" >> "${commentfile}" - - i=0 - until [[ ${i} -eq ${#TP_HEADER[@]} ]]; do - printf "%s\n\n" "${TP_HEADER[${i}]}" >> "${commentfile}" - ((i=i+1)) - done - - { - printf "\n\n" - echo "| Vote | Subsystem | Runtime | Comment |" - echo "|:----:|----------:|--------:|:--------|" - } >> "${commentfile}" - - i=0 - until [[ ${i} -eq ${#TP_VOTE_TABLE[@]} ]]; do - ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') - vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') - comment=$(echo "${ourstring}" | cut -f5 -d\|) - - if [[ "${vote}" = "H" ]]; then - echo "||| _${comment}_ |" >> "${commentfile}" - else - echo "${TP_VOTE_TABLE[${i}]}" >> "${commentfile}" - fi - ((i=i+1)) - done - - if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then - { - printf "\n\n" - echo "| Reason | Tests |" - echo "|-------:|:------|" - } >> "${commentfile}" - i=0 - until [[ ${i} -eq ${#TP_TEST_TABLE[@]} ]]; do - echo "${TP_TEST_TABLE[${i}]}" >> "${commentfile}" - ((i=i+1)) - done - fi - - { - printf "\n\n" - echo "| Subsystem | Report/Notes |" - echo "|----------:|:-------------|" - } >> "${commentfile}" - - i=0 - until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do - comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | - ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") - printf "%s\n" "${comment}" >> "${commentfile}" - ((i=i+1)) - done - - printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" - - github_write_comment "${commentfile}" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/gradle.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/gradle.sh b/precommit/test-patch.d/gradle.sh deleted file mode 100755 index 8c78ab4..0000000 --- a/precommit/test-patch.d/gradle.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# there's nothing in here public, so don't publish docs -# SHELLDOC-IGNORE - -add_build_tool gradle - -declare -a GRADLE_ARGS=() - -function gradle_usage -{ - yetus_add_option "--gradle-cmd=<cmd>" "The 'gradle' command to use (default 'gradle')" - yetus_add_option "--gradlew-cmd=<cmd>" "The 'gradlew' command to use (default 'basedir/gradlew')" -} - -function gradle_parse_args -{ - local i - - for i in "$@"; do - case ${i} in - --gradle-cmd=*) - GRADLE=${i#*=} - ;; - --gradlew-cmd=*) - GRADLEW=${i#*=} - ;; - esac - done - - # if we requested offline, pass that to mvn - if [[ ${OFFLINE} == "true" ]]; then - GRADLE_ARGS=("${GRADLE_ARGS[@]}" --offline) - fi - - GRADLE=${GRADLE:-gradle} - GRADLEW=${GRADLEW:-"${BASEDIR}/gradlew"} -} - -function gradle_precheck -{ - declare gradle_version - if ! verify_command gradle "${GRADLE}"; then - add_vote_table -1 gradle "ERROR: gradle is not available." - return 1 - fi - # finally let folks know what version they'll be dealing with. - gradle_version=$(${GRADLE} --version 2>/dev/null | grep Gradle 2>/dev/null) - add_footer_table gradle "version: ${gradle_version}" - return 0 -} - -function gradle_initialize -{ - if [[ "${BUILDTOOL}" = gradle ]]; then - # shellcheck disable=SC2034 - BUILDTOOLCWD=basedir - fi - - # we need to do this before docker kicks in - if [[ -e "${HOME}/.gradle" - && ! -d "${HOME}/.gradle" ]]; then - yetus_error "ERROR: ${HOME}/.gradle is not a directory." - return 1 - elif [[ ! -e "${HOME}/.gradle" ]]; then - yetus_debug "Creating ${HOME}/.gradle" - mkdir -p "${HOME}/.gradle" - fi -} - -function gradle_filefilter -{ - declare filename=$1 - - if [[ ${filename} =~ build\.gradle$ - || ${filename} =~ gradlew$ - || ${filename} =~ gradle\.properties$ - || ${filename} =~ wrapper\.gradle$ ]]; then - yetus_debug "tests/compile: ${filename}" - add_test compile - fi -} - -function gradle_buildfile -{ - echo "gradlew" -} - -function gradle_executor -{ - echo "${GRADLEW}" "${GRADLE_ARGS[@]}" -} - -## @description Bootstrap gradle -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -function gradle_precompile -{ - declare repostatus=$1 - declare result=0 - - if [[ ${BUILDTOOL} != gradle ]]; then - return 0 - fi - - if [[ "${repostatus}" = branch ]]; then - # shellcheck disable=SC2153 - big_console_header "gradle boostrap: ${PATCH_BRANCH}" - else - big_console_header "gradle bootstrap: ${BUILDMODE}" - fi - - personality_modules "${repostatus}" gradleboot - - pushd "${BASEDIR}" >/dev/null - echo_and_redirect "${PATCH_DIR}/${repostatus}-gradle-bootstrap.txt" gradle -b bootstrap.gradle - popd >/dev/null - - modules_workers "${repostatus}" gradleboot - result=$? - modules_messages "${repostatus}" gradleboot true - if [[ ${result} != 0 ]]; then - return 1 - fi - return 0 -} - -## @description Helper for generic_logfilter -## @audience private -## @stability evolving -## @replaceable no -function gradle_javac_logfilter -{ - declare input=$1 - declare output=$2 - - #shellcheck disable=SC2016,SC2046 - ${GREP} "\.java" "${input}" > "${output}" -} - -## @description Helper for generic_logfilter -## @audience private -## @stability evolving -## @replaceable no -function gradle_javadoc_logfilter -{ - declare input=$1 - declare output=$2 - - #shellcheck disable=SC2016,SC2046 - ${GREP} "javadoc.*\.java" "${input}" > "${output}" -} - -## @description Helper for generic_logfilter -## @audience private -## @stability evolving -## @replaceable no -function gradle_scaladoc_logfilter -{ - declare input=$1 - declare output=$2 - - #shellcheck disable=SC2016,SC2046 - ${GREP} "^\[ant:scaladoc\] /.*\.scala" "${input}" > "${output}" -} - -function gradle_modules_worker -{ - declare repostatus=$1 - declare tst=$2 - shift 2 - - # shellcheck disable=SC2034 - UNSUPPORTED_TEST=false - - case ${tst} in - checkstyle) - modules_workers "${repostatus}" "${tst}" checkstyleMain checkstyleTest - ;; - compile) - modules_workers "${repostatus}" "${tst}" - ;; - distclean) - modules_workers "${repostatus}" clean - ;; - javadoc) - modules_workers "${repostatus}" "${tst}" javadoc - ;; - scaladoc) - modules_workers "${repostatus}" "${tst}" scaladoc - ;; - unit) - modules_workers "${repostatus}" "${tst}" test - ;; - *) - # shellcheck disable=SC2034 - UNSUPPORTED_TEST=true - if [[ ${repostatus} = patch ]]; then - add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" - fi - yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" - return 1 - ;; - esac -} - -function gradle_builtin_personality_modules -{ - local repostatus=$1 - local testtype=$2 - - local module - - yetus_debug "Using builtin personality_modules" - yetus_debug "Personality: ${repostatus} ${testtype}" - - clear_personality_queue - - for module in "${CHANGED_MODULES[@]}"; do - personality_enqueue_module "${module}" - done -} - -function gradle_builtin_personality_file_tests -{ - local filename=$1 - - yetus_debug "Using builtin gradle personality_file_tests" - - if [[ ${filename} =~ src/main/webapp ]]; then - yetus_debug "tests/webapp: ${filename}" - elif [[ ${filename} =~ \.sh - || ${filename} =~ \.cmd - || ${filename} =~ src/main/scripts - || ${filename} =~ src/test/scripts - ]]; then - yetus_debug "tests/shell: ${filename}" - elif [[ ${filename} =~ \.c$ - || ${filename} =~ \.cc$ - || ${filename} =~ \.h$ - || ${filename} =~ \.hh$ - || ${filename} =~ \.proto$ - || ${filename} =~ \.cmake$ - || ${filename} =~ CMakeLists.txt - ]]; then - yetus_debug "tests/units: ${filename}" - add_test cc - add_test unit - elif [[ ${filename} =~ \.scala$ ]]; then - add_test scalac - add_test scaladoc - add_test unit - elif [[ ${filename} =~ build.xml$ - || ${filename} =~ pom.xml$ - || ${filename} =~ \.java$ - ]]; then - yetus_debug "tests/javadoc+units: ${filename}" - add_test javac - add_test javadoc - add_test unit - elif [[ ${filename} =~ src/main ]]; then - yetus_debug "tests/generic+units: ${filename}" - add_test compile - add_test unit - fi - - if [[ ${filename} =~ src/test ]]; then - yetus_debug "tests" - add_test unit - fi - - if [[ ${filename} =~ \.java$ ]]; then - add_test findbugs - fi -} - -function gradle_docker_support -{ - DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${HOME}/.gradle:/home/${USER_NAME}/.gradle") -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/hadolint.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/hadolint.sh b/precommit/test-patch.d/hadolint.sh deleted file mode 100755 index 2392e29..0000000 --- a/precommit/test-patch.d/hadolint.sh +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# no public APIs here -# SHELLDOC-IGNORE - -add_test_type hadolint - -HADOLINT_TIMER=0 -HADOLINT=${HADOLINT:-$(command -v hadolint 2>/dev/null)} - -# files that are going to get hadolint'd -HADOLINT_CHECKFILES=() - -function hadolint_filefilter -{ - declare filename=$1 - - if [[ ${filename} =~ Dockerfile$ ]]; then - add_test hadolint - yetus_add_array_element HADOLINT_CHECKFILES "${filename}" - fi -} - -function hadolint_precheck -{ - declare langs - - if ! verify_command "hadolint" "${HADOLINT}"; then - add_vote_table 0 hadolint "hadolint was not available." - delete_test hadolint - fi - - if [[ -z "${LANG}" ]]; then - langs=$(locale -a) - if [[ ${langs} =~ C.UTF-8 ]]; then - yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing C.UTF-8." - export LANG=C.UTF-8 - export LC_ALL=C.UTF-8 - elif [[ ${langs} =~ en_US.UTF-8 ]]; then - yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing en_US.UTF-8." - export LANG=en_US.UTF-8 - export LC_ALL=en_US.UTF-8 - else - for i in ${langs}; do - if [[ "${i}" =~ UTF-8 ]]; then - yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing ${i}." - export LANG="${i}" - export LC_ALL="${i}" - break - fi - done - fi - fi - - if [[ ! "${LANG}" =~ UTF-8 ]]; then - yetus_error "WARNING: hadolint may fail without UTF-8 locale setting." - fi -} - -function hadolint_logic -{ - declare repostatus=$1 - declare i - - pushd "${BASEDIR}" >/dev/null || return 1 - - for i in "${HADOLINT_CHECKFILES[@]}"; do - if [[ -f "${i}" ]]; then - echo " * ${i}" - "${HADOLINT}" "${i}" >> "${PATCH_DIR}/${repostatus}-hadolint-result.txt" - fi - done - popd > /dev/null || return 1 -} - -function hadolint_preapply -{ - if ! verify_needed_test hadolint; then - return 0 - fi - - big_console_header "hadolint plugin: ${PATCH_BRANCH}" - - start_clock - - hadolint_logic branch - - # keep track of how much as elapsed for us already - HADOLINT_TIMER=$(stop_clock) - return 0 -} - -## filename:line\sCODE Text -function hadolint_calcdiffs -{ - declare branch=$1 - declare patch=$2 - declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} - declare j - - # first, pull out just the errors - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${branch}" | cut -d' ' -f2- > "${tmp}.branch" - - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${patch}" | cut -d' ' -f2- > "${tmp}.patch" - - ${DIFF} --unchanged-line-format="" \ - --old-line-format="" \ - --new-line-format="%dn " \ - "${tmp}.branch" \ - "${tmp}.patch" > "${tmp}.lined" - - # now, pull out those lines of the raw output - # shellcheck disable=SC2013 - for j in $(cat "${tmp}.lined"); do - # shellcheck disable=SC2086 - head -${j} "${patch}" | tail -1 - done - - rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null -} - -function hadolint_postapply -{ - declare i - declare numPrepatch - declare numPostpatch - declare diffPostpatch - declare fixedpatch - declare statstring - - if ! verify_needed_test hadolint; then - return 0 - fi - - big_console_header "hadolint plugin: ${BUILDMODE}" - - start_clock - - # add our previous elapsed to our new timer - # by setting the clock back - offset_clock "${HADOLINT_TIMER}" - - hadolint_logic patch - - calcdiffs \ - "${PATCH_DIR}/branch-hadolint-result.txt" \ - "${PATCH_DIR}/patch-hadolint-result.txt" \ - hadolint \ - > "${PATCH_DIR}/diff-patch-hadolint.txt" - - # shellcheck disable=SC2016 - numPrepatch=$(wc -l "${PATCH_DIR}/branch-hadolint-result.txt" | ${AWK} '{print $1}') - - # shellcheck disable=SC2016 - numPostpatch=$(wc -l "${PATCH_DIR}/patch-hadolint-result.txt" | ${AWK} '{print $1}') - - # shellcheck disable=SC2016 - diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-hadolint.txt" | ${AWK} '{print $1}') - - - ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) - - statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) - - if [[ ${diffPostpatch} -gt 0 ]] ; then - add_vote_table -1 hadolint "${BUILDMODEMSG} ${statstring}" - add_footer_table hadolint "@@BASE@@/diff-patch-hadolint.txt" - bugsystem_linecomments "hadolint" "${PATCH_DIR}/diff-patch-hadolint.txt" - return 1 - elif [[ ${fixedpatch} -gt 0 ]]; then - add_vote_table +1 hadolint "${BUILDMODEMSG} ${statstring}" - return 0 - fi - - add_vote_table +1 hadolint "There were no new hadolint issues." - return 0 -} - -function hadolint_postcompile -{ - declare repostatus=$1 - - if [[ "${repostatus}" = branch ]]; then - hadolint_preapply - else - hadolint_postapply - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/htmlout.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/htmlout.sh b/precommit/test-patch.d/htmlout.sh deleted file mode 100755 index f637d1a..0000000 --- a/precommit/test-patch.d/htmlout.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_bugsystem htmlout - -## @description Usage info for htmlout plugin -## @audience private -## @stability evolving -## @replaceable no -function htmlout_usage -{ - yetus_add_option "--html-report-file=<file>" "Save the final report to an HTML-formated file" -} - -## @description Option parsing for htmlout plugin -## @audience private -## @stability evolving -## @replaceable no -function htmlout_parse_args -{ - declare i - declare fn - - for i in "$@"; do - case ${i} in - --html-report-file=*) - fn=${i#*=} - ;; - esac - done - - if [[ -n "${fn}" ]]; then - if : > "${fn}"; then - HTMLOUT_REPORTFILE_ORIG="${fn}" - HTMLOUT_REPORTFILE=$(yetus_abs "${HTMLOUT_REPORTFILE_ORIG}") - else - yetus_error "WARNING: cannot create HTML report file ${fn}. Ignoring." - fi - fi -} - -## @description Give access to the HTML report file in docker mode -## @audience private -## @stability evolving -## @replaceable no -function htmlout_docker_support -{ - if [[ -n ${HTMLOUT_REPORTFILE} ]]; then - DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${HTMLOUT_REPORTFILE}:/testptch/report.htm") - fi -} - - -## @description Write out an HTML version of the final report to a file -## @audience private -## @stability evolving -## @replaceable no -## @param runresult -function htmlout_finalreport -{ - declare result=$1 - declare i - declare commentfile="${HTMLOUT_REPORTFILE}" - declare comment - declare vote - declare ourstring - declare ela - declare subs - declare color - declare comment - declare calctime - - rm "${commentfile}" 2>/dev/null - - if [[ -z "${HTMLOUT_REPORTFILE}" ]]; then - return - fi - - big_console_header "Writing HTML to ${commentfile}" - - { - echo "<table><tbody>" - - if [[ ${result} == 0 ]]; then - echo "<tr><th><font color=\"green\">+1 overall</font></th></tr>" - else - echo "<tr><th><font color=\"red\">-1 overall</font></th></tr>" - fi - echo "</tbody></table>" - echo "<p></p>" - } > "${commentfile}" - - i=0 - until [[ $i -eq ${#TP_HEADER[@]} ]]; do - ourstring=$(echo "${TP_HEADER[${i}]}" | tr -s ' ') - comment=$(echo "${ourstring}" | cut -f2 -d\|) - printf "<tr><td>%s</td></tr>\n" "${comment}" - ((i=i+1)) - done - - { - echo "<table><tbody>" - echo "<tr>" - echo "<th>Vote</th>" - echo "<th>Subsystem</th>" - echo "<th>Runtime</th>" - echo "<th>Comment</th>" - echo "</tr>" - } >> "${commentfile}" - - i=0 - until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do - ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') - vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') - subs=$(echo "${ourstring}" | cut -f3 -d\|) - ela=$(echo "${ourstring}" | cut -f4 -d\|) - calctime=$(clock_display "${ela}") - comment=$(echo "${ourstring}" | cut -f5 -d\|) - - if [[ "${vote}" = "H" ]]; then - { - echo "<tr>" - printf "\t\t<td></td>" - printf "<td></td>" - printf "<td></td>" - printf "<td><font color=\"%s\">%s</font></td>\n" "brown" "${comment}" - echo "</tr>" - } >> "${commentfile}" - ((i=i+1)) - continue - fi - - # summary line - if [[ -z ${vote} - && -n ${ela} ]]; then - color="black" - elif [[ -z ${vote} ]]; then - # keep same color - true - else - # new vote line - case ${vote} in - 1|"+1") - color="green" - ;; - -1) - color="red" - ;; - 0) - color="blue" - ;; - -0) - color="orange" - ;; - *) - color="black" - ;; - esac - fi - - { - echo "<tr>" - printf "\t\t<td><font color=\"%s\">%s</font></td>" "${color}" "${vote}" - printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}" - printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${calctime}" - printf "<td><font color=\"%s\">%s</font></td>\n" "${color}" "${comment}" - echo "</tr>" - } >> "${commentfile}" - ((i=i+1)) - done - { - echo "</tbody></table>" - echo "<p></p>" - } >> "${commentfile}" - - if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then - { - echo "<table><tbody>" - echo "<tr>" - echo "<th>Reason</th>" - echo "<th>Tests</th>" - echo "</tr>" - } >> "${commentfile}" - - i=0 - until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do - ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') - subs=$(echo "${ourstring}" | cut -f2 -d\|) - comment=$(echo "${ourstring}" | cut -f3 -d\|) - { - echo "<tr>" - printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}" - printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${comment}" - echo "</tr>" - } >> "${commentfile}" - ((i=i+1)) - done - - { - echo "</tbody></table>" - echo "<p></p>" - } >> "${commentfile}" - fi - - { - echo "<table><tbody>" - echo "<tr>" - echo "<th>Subsystem</th>" - echo "<th>Report/Notes</th>" - echo "</tr>" - } >> "${commentfile}" - - i=0 - until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do - ourstring=$(echo "${TP_FOOTER_TABLE[${i}]}" | - ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g" | - tr -s ' ') - subs=$(echo "${ourstring}" | cut -f2 -d\|) - comment=$(echo "${ourstring}" | cut -f3 -d\|) - { - echo "<tr>" - printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}" - printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${comment}" - echo "</tr>" - } >> "${commentfile}" - ((i=i+1)) - done - { - echo "</tbody></table>" - echo "<p></p>" - } >> "${commentfile}" - - printf "<p>This message was automatically generated.</p>" >> "${commentfile}" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/java.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/java.sh b/precommit/test-patch.d/java.sh deleted file mode 100755 index 8515650..0000000 --- a/precommit/test-patch.d/java.sh +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_test_type javac -add_test_type javadoc - -yetus_add_entry JDK_TEST_LIST javadoc - -JAVA_INITIALIZED=false - -function initialize_java -{ - local i - local jdkdir - local tmplist - - if [[ ${JAVA_INITIALIZED} == true ]]; then - return - else - JAVA_INITIALIZED=true - fi - - # if we are in pre-docker mode, don't do any of - # this work since it's all going to be wrong anyway - if [[ "${DOCKERSUPPORT}" = "true" ]]; then - return 0 - fi - - if declare -f maven_add_install >/dev/null 2>&1; then - maven_add_install javac - maven_add_install javadoc - fi - - if [[ -z ${JAVA_HOME:-} ]]; then - case ${OSTYPE} in - Darwin) - if [[ -z "${JAVA_HOME}" ]]; then - if [[ -x /usr/libexec/java_home ]]; then - JAVA_HOME="$(/usr/libexec/java_home)" - export JAVA_HOME - else - export JAVA_HOME=/Library/Java/Home - fi - fi - ;; - *) - yetus_error "WARNING: JAVA_HOME not defined. Disabling java tests." - delete_test javac - delete_test javadoc - return 1 - ;; - esac - fi - - JAVA_HOME=$(yetus_abs "${JAVA_HOME}") - - for i in ${JDK_DIR_LIST}; do - if [[ -d "${i}" ]]; then - jdkdir=$(yetus_abs "${i}") - if [[ ${jdkdir} != "${JAVA_HOME}" ]]; then - tmplist="${tmplist} ${jdkdir}" - fi - else - yetus_error "WARNING: Cannot locate JDK directory ${i}: ignoring" - fi - done - - JDK_DIR_LIST="${tmplist} ${JAVA_HOME}" - JDK_DIR_LIST=${JDK_DIR_LIST/ } -} - -function javac_initialize -{ - initialize_java -} - -function javadoc_initialize -{ - initialize_java -} - -## @description Verify that ${JAVA_HOME} is defined -## @audience public -## @stability stable -## @replaceable no -## @return 1 - no JAVA_HOME -## @return 0 - JAVA_HOME defined -function javac_precheck -{ - declare javaversion - declare listofjdks - declare i - - start_clock - - if [[ -z ${JAVA_HOME:-} ]]; then - yetus_error "ERROR: JAVA_HOME is not defined." - add_vote_table -1 pre-patch "JAVA_HOME is not defined." - return 1 - fi - - javaversion=$(report_jvm_version "${JAVA_HOME}") - add_footer_table "Default Java" "${javaversion}" - - if [[ -n ${JDK_DIR_LIST} - && ${JDK_DIR_LIST} != "${JAVA_HOME}" ]]; then - for i in ${JDK_DIR_LIST}; do - javaversion=$(report_jvm_version "${i}") - listofjdks="${listofjdks} ${i}:${javaversion}" - done - add_footer_table "Multi-JDK versions" "${listofjdks}" - fi - return 0 -} - -function javac_filefilter -{ - declare filename=$1 - - if [[ ${filename} =~ \.java$ ]]; then - yetus_debug "tests/javac: ${filename}" - add_test javac - add_test compile - fi -} - -function javadoc_filefilter -{ - local filename=$1 - - if [[ ${filename} =~ \.java$ ]]; then - yetus_debug "tests/javadoc: ${filename}" - add_test javadoc - fi -} - -## @description -## @audience private -## @stability stable -## @replaceable no -## @return 0 on success -## @return 1 on failure -function javac_compile -{ - declare codebase=$1 - declare multijdkmode=$2 - - if ! verify_needed_test javac; then - return 0 - fi - - if [[ ${codebase} = patch ]]; then - yetus_debug "javac: calling generic_postlog_compare compile javac ${multijdkmode}" - generic_postlog_compare compile javac "${multijdkmode}" - fi -} - -## @description Count and compare the number of JavaDoc warnings pre- and post- patch -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure -function javadoc_rebuild -{ - declare codebase=$1 - declare multijdkmode - - if verify_multijdk_test javadoc; then - multijdkmode=true - else - multijdkmode=false - fi - - if [[ "${codebase}" = branch ]]; then - generic_pre_handler javadoc "${multijdkmode}" - else - generic_post_handler javadoc javadoc "${multijdkmode}" true - fi -} - -## @description Helper for generic_logfilter -## @audience private -## @stability evolving -## @replaceable no -function javac_logfilter -{ - declare input=$1 - declare output=$2 - - #shellcheck disable=SC2016,SC2046 - ${GREP} "^.*.java:[0-9]*:" "${input}" > "${output}" -} - -## @description Helper for generic_logfilter -## @audience private -## @stability evolving -## @replaceable no -function javadoc_logfilter -{ - declare input=$1 - declare output=$2 - - #shellcheck disable=SC2016,SC2046 - ${GREP} "^.*.java:[0-9]*:" "${input}" > "${output}" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/jira.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/jira.sh b/precommit/test-patch.d/jira.sh deleted file mode 100755 index c01587e..0000000 --- a/precommit/test-patch.d/jira.sh +++ /dev/null @@ -1,526 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# no public APIs here -# SHELLDOC-IGNORE - -# this bug system handles JIRA. Personalities -# can override the following variables: - -# base JIRA URL -JIRA_URL=${JIRA_URL:-"https://issues.apache.org/jira"} - -# Issue regex to help identify the project -JIRA_ISSUE_RE='' - -# If the issue status is matched with this pattern, the attached patch is regarded as ready to be applied -JIRA_STATUS_RE='Patch Available' - -add_bugsystem jira - -# Simple function to set a default JIRA user after PROJECT_NAME has been set -function jira_set_jira_user -{ - if [[ -n "${PROJECT_NAME}" && ! "${PROJECT_NAME}" = unknown ]]; then - JIRA_USER=${JIRA_USER:-"${PROJECT_NAME}qa"} - fi -} - -function jira_usage -{ - - jira_set_jira_user - - yetus_add_option "--jira-base-url=<url>" "The URL of the JIRA server (default:'${JIRA_URL}')" - yetus_add_option "--jira-issue-re=<expr>" "Bash regular expression to use when trying to find a jira ref in the patch name (default: '${JIRA_ISSUE_RE}')" - yetus_add_option "--jira-password=<pw>" "The password for accessing JIRA" - yetus_add_option "--jira-status-re=<expr>" "Grep regular expression representing the issue status whose patch is applicable to the codebase (default: '${JIRA_STATUS_RE}')" - yetus_add_option "--jira-user=<user>" "The user to access JIRA command (default: ${JIRA_USER})" -} - -function jira_parse_args -{ - declare i - - jira_set_jira_user - - for i in "$@"; do - case ${i} in - --jira-base-url=*) - JIRA_URL=${i#*=} - ;; - --jira-issue-re=*) - JIRA_ISSUE_RE=${i#*=} - ;; - --jira-password=*) - JIRA_PASSWD=${i#*=} - ;; - --jira-status-re=*) - JIRA_STATUS_RE=${i#*=} - ;; - --jira-user=*) - JIRA_USER=${i#*=} - ;; - esac - done -} - -## @description provides issue determination based upon the URL and more. -## @description WARNING: called from the github plugin! -function jira_determine_issue -{ - declare input=$1 - declare patchnamechunk - declare maybeissue - - if [[ -n ${JIRA_ISSUE} ]]; then - return 0 - fi - - # shellcheck disable=SC2016 - patchnamechunk=$(echo "${input}" | ${AWK} -F/ '{print $NF}') - - maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-) - - if [[ ${maybeissue} =~ ${JIRA_ISSUE_RE} ]]; then - # shellcheck disable=SC2034 - ISSUE=${maybeissue} - JIRA_ISSUE=${maybeissue} - add_footer_table "JIRA Issue" "${JIRA_ISSUE}" - return 0 - fi - - return 1 -} - -function jira_http_fetch -{ - declare input=$1 - declare output=$2 - declare ec - - yetus_debug "jira_http_fetch: ${JIRA_URL}/${input}" - if [[ -n "${JIRA_USER}" - && -n "${JIRA_PASSWD}" ]]; then - ${CURL} --silent --fail \ - --user "${JIRA_USER}:${JIRA_PASSWD}" \ - --output "${output}" \ - --location \ - "${JIRA_URL}/${input}" - else - ${CURL} --silent --fail \ - --output "${output}" \ - --location \ - "${JIRA_URL}/${input}" - fi - ec=$? - case "${ec}" in - "0") - ;; - "1") - yetus_debug "jira_http_fetch: Unsupported protocol. Maybe misspelled jira's url?" - ;; - "3") - yetus_debug "jira_http_fetch: ${JIRA_URL}/${input} url is malformed." - ;; - "6") - yetus_debug "jira_http_fetch: Could not resolve host in URL ${JIRA_URL}." - ;; - "22") - yetus_debug "jira_http_fetch: ${JIRA_URL}/${input} returned 4xx status code. Maybe incorrect username/password?" - ;; - *) - yetus_debug "jira_http_fetch: ${JIRA_URL}/${input} returned $ec error code. See https://ec.haxx.se/usingcurl-returns.html for details." - ;; - esac - return ${ec} -} - -function jira_locate_patch -{ - declare input=$1 - declare fileloc=$2 - declare jsonloc - declare relativeurl - declare retval - declare found=false - - yetus_debug "jira_locate_patch: trying ${JIRA_URL}/browse/${input}" - - if [[ "${OFFLINE}" == true ]]; then - yetus_debug "jira_locate_patch: offline, skipping" - return 1 - fi - - jira_http_fetch "browse/${input}" "${PATCH_DIR}/jira" - if [[ $? != 0 ]]; then - yetus_debug "jira_locate_patch: not a JIRA." - return 1 - fi - - # if github is configured check to see if there is a URL in the text - # that is a github patch file or pull request - if [[ -n "${GITHUB_BASE_URL}" ]]; then - jira_determine_issue "${input}" - # Download information via REST API - jsonloc="${PATCH_DIR}/jira-json" - jira_http_fetch "rest/api/2/issue/${input}" "${jsonloc}" - # Parse the downloaded information to check if the issue is - # just a pointer to GitHub. - github_jira_bridge "${fileloc}" "${jsonloc}" - if [[ $? -eq 0 ]]; then - echo "${input} appears to be a Github PR. Switching Modes." - return 0 - fi - yetus_debug "jira_locate_patch: ${input} seemed like a Github PR, but there was a failure." - fi - - # Not reached if there is a successful github plugin return - if [[ $(${GREP} -c "${JIRA_STATUS_RE}" "${PATCH_DIR}/jira") == 0 ]]; then - if [[ ${ROBOT} == true ]]; then - yetus_error "ERROR: ${input} issue status is not matched with \"${JIRA_STATUS_RE}\"." - cleanup_and_exit 1 - else - yetus_error "WARNING: ${input} issue status is not matched with \"${JIRA_STATUS_RE}\"." - fi - fi - - # See https://jira.atlassian.com/browse/JRA-27637 as why we can't use - # the REST interface here. :( - # the assumption here is that attachment id's are given in an - # ascending order. so bigger # == newer file - #shellcheck disable=SC2016 - tr '>' '\n' < "${PATCH_DIR}/jira" \ - | ${AWK} 'match($0,"/secure/attachment/[0-9]*/[^\"]*"){print substr($0,RSTART,RLENGTH)}' \ - | ${GREP} -v -e 'htm[l]*$' \ - | ${SED} -e 's,[ ]*$,,g' \ - | sort -n -r -k4 -t/ \ - | uniq \ - > "${PATCH_DIR}/jira-attachments.txt" - - echo "${input} patch is being downloaded at $(date) from" - while read -r relativeurl && [[ ${found} = false ]]; do - PATCHURL="${JIRA_URL}${relativeurl}" - - printf " %s -> " "${PATCHURL}" - - jira_http_fetch "${relativeurl}" "${fileloc}" - retval=$? - if [[ ${retval} == 0 ]]; then - found=true - echo "Downloaded" - elif [[ ${retval} == 22 ]]; then - echo "404" - yetus_debug "Presuming the attachment was deleted, trying the next one (see YETUS-298)" - else - echo "Error (curl returned ${retval})" - break - fi - done < <(cat "${PATCH_DIR}/jira-attachments.txt") - - if [[ "${found}" = false ]]; then - yetus_error "ERROR: ${input} could not be downloaded." - cleanup_and_exit 1 - fi - - if [[ ! ${PATCHURL} =~ \.patch$ ]]; then - guess_patch_file "${fileloc}" - if [[ $? == 0 ]]; then - yetus_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. Proceeding, but issue/branch matching might go awry." - add_vote_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${PATCH_NAMING_RULE} for instructions." - else - # this definitely isn't a patch so just bail out. - return 1 - fi - fi - add_footer_table "JIRA Patch URL" "${PATCHURL}" - - return 0 -} - -## @description Try to guess the branch being tested using a variety of heuristics -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success, with PATCH_BRANCH updated appropriately -function jira_determine_branch -{ - declare patchnamechunk - declare total - declare count - declare hinttype - - for hinttype in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do - if [[ -z "${hinttype}" ]]; then - continue - fi - - # If one of these matches the JIRA issue regex - # then we don't want it to trigger the branch - # detection since that's almost certainly not - # intended. In other words, if ISSUE-99 is the - # name of a branch, you want to test ISSUE-99 - # against master, not ISSUE-99's branch - if [[ ${hinttype} =~ ${JIRA_ISSUE_RE} ]]; then - continue - fi - - yetus_debug "Determine branch: starting with ${hinttype}" - patchnamechunk=$(echo "${hinttype}" \ - | ${SED} -e 's,.*/\(.*\)$,\1,' \ - -e 's,\.txt,.,' \ - -e 's,.patch,.,g' \ - -e 's,.diff,.,g' \ - -e 's,\.\.,.,g' \ - -e 's,\.$,,g' ) - - # ISSUE-branch-## - PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-) - yetus_debug "Determine branch: ISSUE-branch-## = ${PATCH_BRANCH}" - if [[ -n "${PATCH_BRANCH}" ]]; then - if verify_valid_branch "${PATCH_BRANCH}"; then - return 0 - fi - fi - - # ISSUE-##[.##].branch - PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d. ) - count="${PATCH_BRANCH//[^.]}" - total=${#count} - ((total = total + 3 )) - until [[ ${total} -lt 3 ]]; do - PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3-${total} -d.) - yetus_debug "Determine branch: ISSUE[.##].branch = ${PATCH_BRANCH}" - ((total=total-1)) - if [[ -n "${PATCH_BRANCH}" ]]; then - if verify_valid_branch "${PATCH_BRANCH}"; then - return 0 - fi - fi - done - - # ISSUE.branch.## - PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2- -d. ) - count="${PATCH_BRANCH//[^.]}" - total=${#count} - ((total = total + 3 )) - until [[ ${total} -lt 2 ]]; do - PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2-${total} -d.) - yetus_debug "Determine branch: ISSUE.branch[.##] = ${PATCH_BRANCH}" - ((total=total-1)) - if [[ -n "${PATCH_BRANCH}" ]]; then - if verify_valid_branch "${PATCH_BRANCH}"; then - return 0 - fi - fi - done - - # ISSUE-branch.## - PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1- -d. ) - count="${PATCH_BRANCH//[^.]}" - total=${#count} - ((total = total + 1 )) - until [[ ${total} -lt 1 ]]; do - PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1-${total} -d. ) - yetus_debug "Determine branch: ISSUE-branch[.##] = ${PATCH_BRANCH}" - ((total=total-1)) - if [[ -n "${PATCH_BRANCH}" ]]; then - if verify_valid_branch "${PATCH_BRANCH}"; then - return 0 - fi - fi - done - done - - return 1 -} - -## @description Write the contents of a file to JIRA -## @param filename -## @stability stable -## @audience public -## @return exit code from posting to jira -function jira_write_comment -{ - declare -r commentfile=${1} - declare retval=0 - - if [[ "${OFFLINE}" == true ]]; then - echo "JIRA Plugin: Running in offline, comment skipped." - return 0 - fi - - if [[ -n ${JIRA_PASSWD} - && -n ${JIRA_USER} ]]; then - - # RESTify the comment - { - echo "{\"body\":\"" - ${SED} -e 's,\\,\\\\,g' \ - -e 's,\",\\\",g' \ - -e 's,$,\\r\\n,g' "${commentfile}" \ - | tr -d '\n' - echo "\"}" - } > "${PATCH_DIR}/jiracomment.$$" - - ${CURL} -X POST \ - -H "Accept: application/json" \ - -H "Content-Type: application/json" \ - -u "${JIRA_USER}:${JIRA_PASSWD}" \ - -d @"${PATCH_DIR}/jiracomment.$$" \ - --silent --location \ - "${JIRA_URL}/rest/api/2/issue/${JIRA_ISSUE}/comment" \ - >/dev/null - retval=$? - rm "${PATCH_DIR}/jiracomment.$$" - else - echo "JIRA Plugin: no credentials provided to write a comment." - fi - return ${retval} -} - -## @description Print out the finished details to the JIRA issue -## @audience private -## @stability evolving -## @replaceable no -## @param runresult -function jira_finalreport -{ - declare result=$1 - declare i - declare commentfile=${PATCH_DIR}/jiracommentfile - declare comment - declare vote - declare ourstring - declare ela - declare subs - declare color - declare comment - declare calctime - - rm "${commentfile}" 2>/dev/null - - if [[ ${ROBOT} == "false" - || ${OFFLINE} == true ]] ; then - return 0 - fi - - if [[ -z "${JIRA_ISSUE}" ]]; then - return 0 - fi - - big_console_header "Adding comment to JIRA" - - if [[ ${result} == 0 ]]; then - echo "| (/) *{color:green}+1 overall{color}* |" >> "${commentfile}" - else - echo "| (x) *{color:red}-1 overall{color}* |" >> "${commentfile}" - fi - - echo "\\\\" >> "${commentfile}" - - i=0 - until [[ $i -eq ${#TP_HEADER[@]} ]]; do - printf "%s\n" "${TP_HEADER[${i}]}" >> "${commentfile}" - ((i=i+1)) - done - - echo "\\\\" >> "${commentfile}" - - echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}" - - i=0 - until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do - ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') - vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') - subs=$(echo "${ourstring}" | cut -f3 -d\|) - ela=$(echo "${ourstring}" | cut -f4 -d\|) - calctime=$(clock_display "${ela}") - comment=$(echo "${ourstring}" | cut -f5 -d\|) - - if [[ "${vote}" = "H" ]]; then - echo "|| || || || {color:brown}${comment}{color} ||" >> "${commentfile}" - ((i=i+1)) - continue - fi - - # summary line - if [[ -z ${vote} - && -n ${ela} ]]; then - color="black" - elif [[ -z ${vote} ]]; then - # keep same color - true - else - # new vote line - case ${vote} in - 1|"+1") - color="green" - ;; - -1) - color="red" - ;; - 0) - color="blue" - ;; - -0) - color="orange" - ;; - H) - # this never gets called (see above) but this is here so others know the color is taken - color="brown" - ;; - *) - color="black" - ;; - esac - fi - - printf "| {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} |\n" \ - "${color}" "${vote}" \ - "${color}" "${subs}" \ - "${color}" "${calctime}" \ - "${color}" "${comment}" \ - >> "${commentfile}" - ((i=i+1)) - done - - if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then - { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" - - echo "|| Reason || Tests ||" >> "${commentfile}" - i=0 - until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do - printf "%s\n" "${TP_TEST_TABLE[${i}]}" >> "${commentfile}" - ((i=i+1)) - done - fi - - { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" - - echo "|| Subsystem || Report/Notes ||" >> "${commentfile}" - i=0 - until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do - comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | - ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g") - printf "%s\n" "${comment}" >> "${commentfile}" - ((i=i+1)) - done - - printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" - - jira_write_comment "${commentfile}" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/junit.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/junit.sh b/precommit/test-patch.d/junit.sh deleted file mode 100755 index 0bbe23e..0000000 --- a/precommit/test-patch.d/junit.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_test_format junit - -JUNIT_TEST_TIMEOUTS="" -JUNIT_FAILED_TESTS="" - -JUNIT_TEST_OUTPUT_DIR="." -JUNIT_TEST_PREFIX="org.apache." - -function junit_usage -{ - yetus_add_option "--junit-test-output=<path>" "Directory to search for the test output TEST-*.xml files, relative to the module directory (default:'${JUNIT_TEST_OUTPUT_DIR}')" - yetus_add_option "--junit-test-prefix=<prefix to trim>" "Prefix of test names to be be removed. Used to shorten test names by removing common package name. (default:'${JUNIT_TEST_PREFIX}')" -} - -function junit_parse_args -{ - declare i - - for i in "$@"; do - case ${i} in - --junit-test-output=*) - JUNIT_TEST_OUTPUT_DIR=${i#*=} - ;; - --junit-test-prefix=*) - JUNIT_TEST_PREFIX=${i#*=} - ;; - esac - done -} - -function junit_process_tests -{ - # shellcheck disable=SC2034 - declare module=$1 - declare buildlogfile=$2 - declare result=0 - declare module_test_timeouts - declare module_failed_tests - - # shellcheck disable=SC2016 - module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${buildlogfile}") - if [[ -n "${module_test_timeouts}" ]] ; then - JUNIT_TEST_TIMEOUTS="${JUNIT_TEST_TIMEOUTS} ${module_test_timeouts}" - ((result=result+1)) - fi - - #shellcheck disable=SC2026,SC2038,SC2016 - module_failed_tests=$(find "${JUNIT_TEST_OUTPUT_DIR}" -name 'TEST*.xml'\ - | xargs "${GREP}" -l -E "<failure|<error"\ - | ${AWK} -F/ '{sub("'"TEST-${JUNIT_TEST_PREFIX}"'",""); sub(".xml",""); print $NF}') - if [[ -n "${module_failed_tests}" ]] ; then - JUNIT_FAILED_TESTS="${JUNIT_FAILED_TESTS} ${module_failed_tests}" - ((result=result+1)) - fi - - if [[ ${result} -gt 0 ]]; then - return 1 - fi - return 0 -} - -function junit_finalize_results -{ - declare jdk=$1 - - if [[ -n "${JUNIT_FAILED_TESTS}" ]] ; then - # shellcheck disable=SC2086 - populate_test_table "${jdk}Failed junit tests" ${JUNIT_FAILED_TESTS} - JUNIT_FAILED_TESTS="" - fi - if [[ -n "${JUNIT_TEST_TIMEOUTS}" ]] ; then - # shellcheck disable=SC2086 - populate_test_table "${jdk}Timed out junit tests" ${JUNIT_TEST_TIMEOUTS} - JUNIT_TEST_TIMEOUTS="" - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/test-patch.d/make.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/make.sh b/precommit/test-patch.d/make.sh deleted file mode 100755 index 6fb2b59..0000000 --- a/precommit/test-patch.d/make.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRCMAKEIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_build_tool make - -MAKE=make -MAKEFILE=Makefile - -## @description make usage hook -## @audience private -## @stability evolving -## @replaceable no -function make_usage -{ - yetus_add_option "--make-cmd=<cmd>" "The 'make' command to use (default: '${MAKE}')" - yetus_add_option "--make-file=<filename>" "The name of the file the make cmd should work on (default: '${MAKEFILE}')" -} - -## @description precheck make -## @audience private -## @stability evolving -## @replaceable no -function make_precheck -{ - declare make_version - if ! verify_command make "${MAKE}"; then - add_vote_table -1 make "make was not available." - return 1 - fi - # finally let folks know what version they'll be dealing with. - if make_version=$(set -o pipefail; ${MAKE} --version 2>/dev/null | head -n 1 2>/dev/null) && [ -n "${make_version}" ]; then - add_footer_table make "version: ${make_version}" - fi - return 0 -} - -## @description make argument parser -## @audience private -## @stability evolving -## @param args -function make_parse_args -{ - declare i - - for i in "$@"; do - case ${i} in - --make-cmd=*) - MAKE=${i#*=} - ;; - --make-file=*) - MAKEFILE=${i#*=} - ;; - --make-use-git-clean) - MAKE_GITCLEAN=true - ;; - esac - done -} - -## @description get the name of the make build filename -## @audience private -## @stability evolving -## @replaceable no -## @return make build file -function make_buildfile -{ - echo "${MAKEFILE}" -} - -## @description get the name of the make binary -## @audience private -## @stability evolving -## @replaceable no -## @return filename -## @param testname -function make_executor -{ - echo "${MAKE} ${MAKE_ARGS[*]}" -} - -## @description make worker -## @audience private -## @stability evolving -## @replaceable no -## @return status -## @param repostatus -## @param test -function make_modules_worker -{ - declare repostatus=$1 - declare tst=$2 - shift 2 - - # shellcheck disable=SC2034 - UNSUPPORTED_TEST=false - - case ${tst} in - compile) - modules_workers "${repostatus}" "${tst}" - ;; - distclean) - if [[ ${MAKE_GITCLEAN} = true ]];then - git clean -x -f -d - else - modules_workers "${repostatus}" distclean clean - fi - ;; - unit) - modules_workers "${repostatus}" test test - ;; - *) - # shellcheck disable=SC2034 - UNSUPPORTED_TEST=true - if [[ ${repostatus} = patch ]]; then - add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" - fi - yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" - return 1 - ;; - esac -} - -## @description make module queuer -## @audience private -## @stability evolving -## @replaceable no -function make_builtin_personality_modules -{ - declare repostatus=$1 - declare testtype=$2 - - declare module - - yetus_debug "Using builtin personality_modules" - yetus_debug "Personality: ${repostatus} ${testtype}" - - clear_personality_queue - - for module in "${CHANGED_MODULES[@]}"; do - personality_enqueue_module "${module}" - done -} - -## @description make test determiner -## @audience private -## @stability evolving -## @replaceable no -## @param filename -function make_builtin_personality_file_tests -{ - declare filename=$1 - - yetus_debug "Using builtin make personality_file_tests" - - if [[ ${filename} =~ \.c$ - || ${filename} =~ \.cc$ - || ${filename} =~ \.h$ - || ${filename} =~ \.hh$ - || ${filename} =~ \.hh\.in$ - || ${filename} =~ \.proto$ - || ${filename} =~ \.make$ - || ${filename} =~ Makefile$ - ]]; then - yetus_debug "tests/units: ${filename}" - add_test compile - add_test unit - fi -}
