http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/author.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/author.sh b/precommit/test-patch.d/author.sh new file mode 100755 index 0000000..d456bbb --- /dev/null +++ b/precommit/test-patch.d/author.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type author + +## @description Check the current directory for @author tags +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function author_patchfile +{ + declare patchfile=$1 + declare authorTags + # shellcheck disable=SC2155 + declare -r appname=$(basename "${BASH_SOURCE-$0}") + + big_console_header "Checking there are no @author tags in the patch." + + start_clock + + if [[ ${CHANGED_FILES} =~ ${appname} ]]; then + echo "Skipping @author checks as ${appname} has been patched." + add_vote_table 0 @author "Skipping @author checks as ${appname} has been patched." + return 0 + fi + + authorTags=$("${GREP}" -c -i '^[^-].*@author' "${patchfile}") + echo "There appear to be ${authorTags} @author tags in the patch." + if [[ ${authorTags} != 0 ]] ; then + add_vote_table -1 @author \ + "The patch appears to contain ${authorTags} @author tags which the" \ + " community has agreed to not allow in code contributions." + return 1 + fi + add_vote_table +1 @author "The patch does not contain any @author tags." + return 0 +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/cc.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/cc.sh b/precommit/test-patch.d/cc.sh new file mode 100755 index 0000000..cae91d3 --- /dev/null +++ b/precommit/test-patch.d/cc.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type cc + +function cc_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.cpp$ + || ${filename} =~ \.cxx$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + ]]; then + yetus_debug "tests/cc: ${filename}" + add_test cc + add_test compile + fi +} + +## @description check for C/C++ compiler errors +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function cc_compile +{ + declare codebase=$1 + declare multijdkmode=$2 + + verify_needed_test cc + if [[ $? = 0 ]]; then + return 0 + fi + + if [[ ${codebase} = patch ]]; then + generic_postlog_compare compile cc "${multijdkmode}" + fi +} + +function cc_count_probs +{ + declare warningfile=$1 + + #shellcheck disable=SC2016,SC2046 + ${GREP} -E '^.*\.(c|cc|h|hh)\:[[:digit:]]*\:' "${warningfile}" | ${AWK} '{sum+=1} END {print sum}' +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/checkstyle.sh b/precommit/test-patch.d/checkstyle.sh new file mode 100755 index 0000000..24b9ffc --- /dev/null +++ b/precommit/test-patch.d/checkstyle.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type checkstyle + +CHECKSTYLE_TIMER=0 + +function checkstyle_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} == maven + || ${BUILDTOOL} == ant ]]; then + if [[ ${filename} =~ \.java$ ]]; then + add_test checkstyle + fi + fi +} + +function checkstyle_runner +{ + local repostatus=$1 + local tmp=${PATCH_DIR}/$$.${RANDOM} + local j + local i=0 + local fn + local savestart=${TIMER} + local savestop + local output + local logfile + local repo + local modulesuffix + local cmd + + modules_reset + + if [[ ${repostatus} == branch ]]; then + repo=${PATCH_BRANCH} + else + repo="the patch" + fi + + #shellcheck disable=SC2153 + until [[ $i -eq ${#MODULE[@]} ]]; do + start_clock + fn=$(module_file_fragment "${MODULE[${i}]}") + modulesuffix=$(basename "${MODULE[${i}]}") + output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt" + logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt" + + if [[ ${BUILDTOOLCWD} == true ]]; then + pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null + fi + + case ${BUILDTOOL} in + ant) + cmd="${ANT} \ + -Dcheckstyle.consoleOutput=true \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \ + ${ANT_ARGS[*]} checkstyle" + ;; + maven) + cmd="${MAVEN} ${MAVEN_ARGS[*]} \ + checkstyle:checkstyle \ + -Dcheckstyle.consoleOutput=true \ + ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch" + ;; + *) + UNSUPPORTED_TEST=true + return 0 + ;; + esac + + #shellcheck disable=SC2086 + echo ${cmd} "> ${logfile}" + #shellcheck disable=SC2086 + ${cmd} 2>&1 \ + | tee "${logfile}" \ + | ${GREP} ^/ \ + | ${SED} -e "s,${BASEDIR},.,g" \ + > "${tmp}" + + if [[ $? == 0 ]] ; then + module_status ${i} +1 "${logfile}" "${modulesuffix} in ${repo} passed checkstyle" + else + module_status ${i} -1 "${logfile}" "${modulesuffix} in ${repo} failed checkstyle" + ((result = result + 1)) + fi + savestop=$(stop_clock) + #shellcheck disable=SC2034 + MODULE_STATUS_TIMER[${i}]=${savestop} + + for j in ${CHANGED_FILES}; do + ${GREP} "${j}" "${tmp}" >> "${output}" + done + + rm "${tmp}" 2>/dev/null + + if [[ ${BUILDTOOLCWD} == true ]]; then + popd >/dev/null + fi + ((i=i+1)) + done + + TIMER=${savestart} + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +function checkstyle_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + checkstyle_preapply + else + checkstyle_postapply + fi +} + +function checkstyle_preapply +{ + local result + + verify_needed_test checkstyle + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "${PATCH_BRANCH} checkstyle" + + start_clock + + personality_modules branch checkstyle + checkstyle_runner branch + result=$? + modules_messages branch checkstyle true + + # keep track of how much as elapsed for us already + CHECKSTYLE_TIMER=$(stop_clock) + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function checkstyle_postapply +{ + local result + local module + local mod + local fn + local i=0 + local numprepatch=0 + local numpostpatch=0 + local diffpostpatch=0 + + verify_needed_test checkstyle + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "Patch checkstyle plugin" + + start_clock + + personality_modules patch checkstyle + checkstyle_runner patch + result=$? + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${CHECKSTYLE_TIMER}" + + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + module=${MODULE[$i]} + fn=$(module_file_fragment "${module}") + + if [[ ! -f "${PATCH_DIR}/branch-checkstyle-${fn}.txt" ]]; then + touch "${PATCH_DIR}/branch-checkstyle-${fn}.txt" + fi + + calcdiffs "${PATCH_DIR}/branch-checkstyle-${fn}.txt" "${PATCH_DIR}/patch-checkstyle-${fn}.txt" > "${PATCH_DIR}/diff-checkstyle-${fn}.txt" + #shellcheck disable=SC2016 + diffpostpatch=$(wc -l "${PATCH_DIR}/diff-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + + if [[ ${diffpostpatch} -gt 0 ]] ; then + ((result = result + 1)) + + # shellcheck disable=SC2016 + numprepatch=$(wc -l "${PATCH_DIR}/branch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + # shellcheck disable=SC2016 + numpostpatch=$(wc -l "${PATCH_DIR}/patch-checkstyle-${fn}.txt" | ${AWK} '{print $1}') + + mod=${module} + if [[ ${mod} == . ]]; then + mod=root + fi + module_status ${i} -1 "diff-checkstyle-${fn}.txt" "Patch generated "\ + "${diffpostpatch} new checkstyle issues in "\ + "${mod} (total was ${numprepatch}, now ${numpostpatch})." + fi + ((i=i+1)) + done + + modules_messages patch checkstyle true + + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/findbugs.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/findbugs.sh b/precommit/test-patch.d/findbugs.sh new file mode 100755 index 0000000..ae18893 --- /dev/null +++ b/precommit/test-patch.d/findbugs.sh @@ -0,0 +1,411 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FINDBUGS_HOME=${FINDBUGS_HOME:-} +FINDBUGS_WARNINGS_FAIL_PRECHECK=false + +add_test_type findbugs + +function findbugs_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} == maven + || ${BUILDTOOL} == ant ]]; then + if [[ ${filename} =~ \.java$ + || ${filename} =~ (^|/)findbugs-exclude.xml$ ]]; then + add_test findbugs + fi + fi +} + +function findbugs_usage +{ + echo "FindBugs specific:" + echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail" +} + +function findbugs_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} + ;; + --findbugs-strict-precheck) + FINDBUGS_WARNINGS_FAIL_PRECHECK=true + ;; + esac + done +} + +## @description are the needed bits for findbugs present? +## @audience private +## @stability evolving +## @replaceable no +## @return 0 findbugs will work for our use +## @return 1 findbugs is missing some component +function findbugs_is_installed +{ + declare exec + declare status=0 + + for exec in findbugs \ + computeBugHistory \ + convertXmlToText \ + filterBugs \ + setBugDatabaseInfo; do + if [[ ! -x "${FINDBUGS_HOME}/bin/${exec}" ]]; then + yetus_error "ERROR: ${FINDBUGS_HOME}/bin/${exec} is not executable." + status=1 + fi + done + return ${status} +} + +## @description Run the maven findbugs plugin and record found issues in a bug database +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function findbugs_runner +{ + local name=$1 + local module + local result=0 + local fn + local warnings_file + local i=0 + local savestop + + personality_modules "${name}" findbugs + "${BUILDTOOL}_modules_worker" "${name}" findbugs + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + #shellcheck disable=SC2153 + until [[ ${i} -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + start_clock + offset_clock "${MODULE_STATUS_TIMER[${i}]}" + module="${MODULE[${i}]}" + fn=$(module_file_fragment "${module}") + + case ${BUILDTOOL} in + maven) + file="${module}/target/findbugsXml.xml" + ;; + ant) + file="${ANT_FINDBUGSXML}" + ;; + esac + + + if [[ ! -f ${file} ]]; then + module_status ${i} -1 "" "${name}/${module} no findbugs output file (${file})" + ((i=i+1)) + continue + fi + + warnings_file="${PATCH_DIR}/${name}-findbugs-${fn}-warnings" + + cp -p "${file}" "${warnings_file}.xml" + + if [[ ${name} == branch ]]; then + "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${PATCH_BRANCH}" \ + "${warnings_file}.xml" "${warnings_file}.xml" + else + "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name patch \ + "${warnings_file}.xml" "${warnings_file}.xml" + fi + if [[ $? != 0 ]]; then + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + module_status ${i} -1 "" "${name}/${module} cannot run setBugDatabaseInfo from findbugs" + ((result=result+1)) + ((i=i+1)) + continue + fi + + "${FINDBUGS_HOME}/bin/convertXmlToText" -html \ + "${warnings_file}.xml" \ + "${warnings_file}.html" + if [[ $? != 0 ]]; then + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + module_status ${i} -1 "" "${name}/${module} cannot run convertXmlToText from findbugs" + ((result=result+1)) + fi + + if [[ -z ${FINDBUGS_VERSION} + && ${name} == branch ]]; then + FINDBUGS_VERSION=$(${GREP} -i "BugCollection version=" "${warnings_file}.xml" \ + | cut -f2 -d\" \ + | cut -f1 -d\" ) + if [[ -n ${FINDBUGS_VERSION} ]]; then + add_footer_table findbugs "v${FINDBUGS_VERSION}" + fi + fi + + ((i=i+1)) + done + return ${result} +} + +## @description Track pre-existing findbugs warnings +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function findbugs_preapply +{ + local fn + local module + local i=0 + local warnings_file + local module_findbugs_warnings + local result=0 + + verify_needed_test findbugs + + if [[ $? == 0 ]]; then + return 0 + fi + + findbugs_is_installed + if [[ $? != 0 ]]; then + add_vote_table 0 findbugs "findbugs executables are not available." + return 0 + fi + + big_console_header "Pre-patch findbugs detection" + + findbugs_runner branch + result=$? + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + module=${MODULE[${i}]} + start_clock + offset_clock "${MODULE_STATUS_TIMER[${i}]}" + fn=$(module_file_fragment "${module}") + warnings_file="${PATCH_DIR}/branch-findbugs-${fn}-warnings" + # shellcheck disable=SC2016 + module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \ + "${PATCH_BRANCH}" \ + "${warnings_file}.xml" \ + "${warnings_file}.xml" \ + | ${AWK} '{print $1}') + + if [[ ${module_findbugs_warnings} -gt 0 ]] ; then + module_status ${i} -1 "branch-findbugs-${fn}.html" "${module} in ${PATCH_BRANCH} cannot run convertXmlToText from findbugs" + ((result=result+1)) + fi + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + done + modules_messages branch findbugs true + fi + + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description Verify patch does not trigger any findbugs warnings +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function findbugs_postinstall +{ + local module + local fn + local combined_xml + local branchxml + local patchxml + local newbugsbase + local fixedbugsbase + local new_findbugs_warnings + local fixed_findbugs_warnings + local line + local firstpart + local secondpart + local i=0 + local result=0 + local savestop + + verify_needed_test findbugs + + if [[ $? == 0 ]]; then + return 0 + fi + + findbugs_is_installed + if [[ $? != 0 ]]; then + return 0 + fi + + big_console_header "Patch findbugs detection" + + findbugs_runner patch + + if [[ ${UNSUPPORTED_TEST} = true ]]; then + return 0 + fi + + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + start_clock + offset_clock "${MODULE_STATUS_TIMER[${i}]}" + module="${MODULE[${i}]}" + + if [[ ${BUILDTOOLCWD} == true ]]; then + pushd "${module}" >/dev/null + fi + + fn=$(module_file_fragment "${module}") + + combined_xml="${PATCH_DIR}/combined-findbugs-${fn}.xml" + branchxml="${PATCH_DIR}/branch-findbugs-${fn}-warnings.xml" + patchxml="${PATCH_DIR}/patch-findbugs-${fn}-warnings.xml" + + if [[ ! -f "${branchxml}" ]]; then + branchxml=${patchxml} + fi + + newbugsbase="${PATCH_DIR}/new-findbugs-${fn}" + fixedbugsbase="${PATCH_DIR}/fixed-findbugs-${fn}" + + "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \ + -output "${combined_xml}" \ + "${branchxml}" \ + "${patchxml}" + if [[ $? != 0 ]]; then + if [[ ${BUILDTOOLCWD} == true ]]; then + popd >/dev/null + fi + module_status ${i} -1 "" "${module} cannot run computeBugHistory from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + #shellcheck disable=SC2016 + new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \ + "${combined_xml}" "${newbugsbase}.xml" | ${AWK} '{print $1}') + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run filterBugs (#1) from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + #shellcheck disable=SC2016 + fixed_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \ + "${combined_xml}" "${fixedbugsbase}.xml" | ${AWK} '{print $1}') + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run filterBugs (#2) from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${fixed_findbugs_warnings} newly fixed warnings." + + "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newbugsbase}.xml" \ + "${newbugsbase}.html" + if [[ $? != 0 ]]; then + popd >/dev/null + module_status ${i} -1 "" "${module} cannot run convertXmlToText from findbugs" + ((result=result+1)) + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + ((i=i+1)) + continue + fi + + if [[ ${new_findbugs_warnings} -gt 0 ]] ; then + populate_test_table FindBugs "module:${module}" + while read line; do + firstpart=$(echo "${line}" | cut -f2 -d:) + secondpart=$(echo "${line}" | cut -f9- -d' ') + add_test_table "" "${firstpart}:${secondpart}" + done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newbugsbase}.xml") + + module_status ${i} -1 "new-findbugs-${fn}.html" "${module} introduced "\ + "${new_findbugs_warnings} new FindBugs issues." + ((result=result+1)) + fi + savestop=$(stop_clock) + MODULE_STATUS_TIMER[${i}]=${savestop} + popd >/dev/null + ((i=i+1)) + done + + modules_messages patch findbugs true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function findbugs_rebuild +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + findbugs_preapply + else + findbugs_postinstall + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/github.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/github.sh b/precommit/test-patch.d/github.sh new file mode 100755 index 0000000..468a228 --- /dev/null +++ b/precommit/test-patch.d/github.sh @@ -0,0 +1,492 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This bug system provides github integration + +add_bugsystem github + +# personalities can override the following settings: + +# Web interface URL. +GITHUB_BASE_URL="https://github.com"; + +# API interface URL. +GITHUB_API_URL="https://api.github.com"; + +# user/repo +GITHUB_REPO="" + +# user settings +GITHUB_PASSWD="" +GITHUB_TOKEN="" +GITHUB_USER="" +GITHUB_ISSUE="" + +# private globals... +GITHUB_BRIDGED=false +GITHUB_COMMITSHA="" + +function github_usage +{ + echo "GITHUB Options:" + echo "--github-api-url=<url> The URL of the API for github (default: '${GITHUB_API_URL}')" + echo "--github-base-url=<url> The URL of the github server (default:'${GITHUB_BASE_URL}')" + echo "--github-password=<pw> Github password" + echo "--github-repo=<repo> github repo to use (default:'${GITHUB_REPO}')" + echo "--github-token=<token> The token to use to write to github" + echo "--github-user=<user> Github user" +} + +function github_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --github-api-url=*) + GITHUB_API_URL=${i#*=} + ;; + --github-base-url=*) + GITHUB_BASE_URL=${i#*=} + ;; + --github-repo=*) + GITHUB_REPO=${i#*=} + ;; + --github-token=*) + GITHUB_TOKEN=${i#*=} + ;; + --github-password=*) + GITHUB_PASSWD=${i#*=} + ;; + --github-user=*) + GITHUB_USER=${i#*=} + ;; + esac + done +} + +## @description this gets called when JIRA thinks this +## @description issue is just a pointer to github +## @description WARNING: Called from JIRA plugin! +function github_jira_bridge +{ + declare fileloc=$1 + declare urlfromjira + + # we use this to prevent loops later on + GITHUB_BRIDGED=true + + # the JIRA issue has already been downloaded. So let's find the URL. + # shellcheck disable=SC2016 + urlfromjira=$(${AWK} "match(\$0,\"${GITHUB_BASE_URL}/[^ ]*patch\"){print substr(\$0,RSTART,RLENGTH)}" "${PATCH_DIR}/jira" | tail -1) + github_breakup_url "${urlfromjira}" + github_locate_patch "${GITHUB_ISSUE}" "${fileloc}" +} + +## @description given a URL, break it up into github plugin globals +## @description this will *override* any personality or yetus defaults +## @params url +function github_breakup_url +{ + declare url=$1 + declare count + declare pos1 + declare pos2 + + count=${url//[^\/]} + count=${#count} + ((pos2=count-3)) + ((pos1=pos2)) + + GITHUB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/) + + ((pos1=pos1+1)) + ((pos2=pos1+1)) + + GITHUB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/) + + ((pos1=pos2+2)) + unset pos2 + + GITHUB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.) +} + + +## @description based upon a github PR, attempt to link back to JIRA +function github_find_jira_title +{ + declare title + declare maybe + declare retval + + if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then + return 1 + fi + + title=$(${GREP} title "${PATCH_DIR}/github-pull.json" \ + | cut -f4 -d\") + + # people typically do two types: JIRA-ISSUE: and [JIRA-ISSUE] + # JIRA_ISSUE_RE is pretty strict so we need to chop that stuff + # out first + + maybe=$(echo "${title}" | cut -f2 -d\[ | cut -f1 -d\]) + jira_determine_issue "${maybe}" + retval=$? + + if [[ ${retval} == 0 ]]; then + return 0 + fi + + maybe=$(echo "${title}" | cut -f1 -d:) + jira_determine_issue "${maybe}" + retval=$? + + if [[ ${retval} == 0 ]]; then + return 0 + fi + + return 1 +} + +function github_determine_issue +{ + declare input=$1 + + if [[ ${input} =~ ^[0-9]+$ + && -n ${GITHUB_REPO} ]]; then + # shellcheck disable=SC2034 + ISSUE=${input} + if [[ -z ${GITHUB_ISSUE} ]]; then + GITHUB_ISSUE=${input} + fi + fi + + # if JIRA didn't call us, should we call it? + if [[ ${GITHUB_BRIDGED} == false ]]; then + github_find_jira_title + if [[ $? == 0 ]]; then + return 0 + fi + fi + + if [[ -n ${GITHUB_ISSUE} ]]; then + return 0 + fi + + return 1 +} + +## @description Try to guess the branch being tested using a variety of heuristics +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success, with PATCH_BRANCH updated appropriately +## @return 1 on failure +function github_determine_branch +{ + if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then + return 1 + fi + + # shellcheck disable=SC2016 + PATCH_BRANCH=$(${AWK} 'match($0,"\"ref\": \""){print $2}' "${PATCH_DIR}/github-pull.json"\ + | cut -f2 -d\"\ + | tail -1 ) + + yetus_debug "Github determine branch: starting with ${PATCH_BRANCH}" + + verify_valid_branch "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return 0 + fi + return 1 +} + +function github_locate_patch +{ + declare input=$1 + declare output=$2 + declare githubauth + + if [[ "${OFFLINE}" == true ]]; then + yetus_debug "github_locate_patch: offline, skipping" + return 1 + fi + + + # https://github.com/your/repo/pull/## + if [[ ${input} =~ ^${GITHUB_BASE_URL}.*/pull/[0-9]+$ ]]; then + github_breakup_url "${input}.patch" + input=${GITHUB_ISSUE} + fi + + # https://github.com/your/repo/pulls/##.patch + if [[ ${input} =~ ^${GITHUB_BASE_URL}.*patch$ ]]; then + github_breakup_url "${input}" + input=${GITHUB_ISSUE} + fi + + # https://github.com/your/repo/pulls/##.diff + if [[ ${input} =~ ^${GITHUB_BASE_URL}.*diff$ ]]; then + github_breakup_url "${input}" + input=${GITHUB_ISSUE} + fi + + # if it isn't a number at this point, no idea + # how to process + if [[ ! ${input} =~ ^[0-9]+$ ]]; then + yetus_debug "github: ${input} is not a pull request #" + return 1 + fi + + # we always pull the .patch version (even if .diff was given) + # with the assumption that this way binary files work. + # The downside of this is that the patch files are + # significantly larger and therefore take longer to process + PATCHURL="${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}.patch" + echo "GITHUB PR #${input} is being downloaded at $(date) from" + echo "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}" + + if [[ -n "${GITHUB_USER}" + && -n "${GITHUB_PASSWD}" ]]; then + githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" + elif [[ -n "${GITHUB_TOKEN}" ]]; then + githubauth="Authorization: token ${GITHUB_TOKEN}" + else + githubauth="X-ignore-me: fake" + fi + + # Let's pull the PR JSON for later use + ${CURL} --silent --fail \ + -H "Accept: application/vnd.github.v3.full+json" \ + -H "${githubauth}" \ + --output "${PATCH_DIR}/github-pull.json" \ + --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${input}" + + echo "Patch from GITHUB PR #${input} is being downloaded at $(date) from" + echo "${PATCHURL}" + + # the actual patch file + ${CURL} --silent --fail \ + --output "${output}" \ + --location \ + -H "${githubauth}" \ + "${PATCHURL}" + + if [[ $? != 0 ]]; then + yetus_debug "github_locate_patch: not a github pull request." + return 1 + fi + + GITHUB_ISSUE=${input} + + # github will translate this to be #(xx) ! + add_footer_table "GITHUB PR" "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}" + + return 0 +} + +function github_linecomments +{ + declare plugin=$1 + declare file=$2 + # shellcheck disable=SC2034 + declare realline=$3 + declare uniline=$4 + declare text=$5 + declare tempfile="${PATCH_DIR}/ghcomment.$$.${RANDOM}" + declare githubauth + + if [[ "${file}" =~ ^./ ]]; then + file=${file##./} + fi + + if [[ -z "${GITHUB_COMMITSHA}" ]]; then + GITHUB_COMMITSHA=$(${GREP} \"sha\" "${PATCH_DIR}/github-pull.json" 2>/dev/null \ + | head -1 \ + | cut -f4 -d\") + fi + + if [[ -z "${uniline}" ]]; then + return + fi + + # build our REST post + { + printf "{\"body\":\"" + echo "${plugin}: ${text}" \ + | ${SED} -e 's,\\,\\\\,g' \ + -e 's,\",\\\",g' \ + -e 's,$,\\r\\n,g' \ + | tr -d '\n' + echo "\"," + echo "\"commit_id\":\"${GITHUB_COMMITSHA}\"," + echo "\"path\":\"${file}\"," + echo "\"position\":${uniline}" + echo "}" + } > "${tempfile}" + + if [[ -n "${GITHUB_USER}" + && -n "${GITHUB_PASSWD}" ]]; then + githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" + elif [[ -n "${GITHUB_TOKEN}" ]]; then + githubauth="Authorization: token ${GITHUB_TOKEN}" + else + return 0 + fi + + ${CURL} -X POST \ + -H "Accept: application/vnd.github.v3.full+json" \ + -H "Content-Type: application/json" \ + -H "${githubauth}" \ + -d @"${tempfile}" \ + --silent --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${GITHUB_ISSUE}/comments" \ + >/dev/null + rm "${tempfile}" +} + +## @description Write the contents of a file to github +## @params filename +## @stability stable +## @audience public +function github_write_comment +{ + declare -r commentfile=${1} + declare retval=0 + declare restfile="${PATCH_DIR}/ghcomment.$$" + declare githubauth + + if [[ "${OFFLINE}" == true ]]; then + echo "Github Plugin: Running in offline, comment skipped." + return 0 + fi + + { + printf "{\"body\":\"" + ${SED} -e 's,\\,\\\\,g' \ + -e 's,\",\\\",g' \ + -e 's,$,\\r\\n,g' "${commentfile}" \ + | tr -d '\n' + echo "\"}" + } > "${restfile}" + + if [[ -n "${GITHUB_USER}" + && -n "${GITHUB_PASSWD}" ]]; then + githubauth="${GITHUB_USER}:${GITHUB_PASSWD}" + elif [[ -n "${GITHUB_TOKEN}" ]]; then + githubauth="Authorization: token ${GITHUB_TOKEN}" + else + echo "Github Plugin: no credentials provided to write a comment." + return 0 + fi + + ${CURL} -X POST \ + -H "Accept: application/vnd.github.v3.full+json" \ + -H "Content-Type: application/json" \ + -H "${githubauth}" \ + -d @"${restfile}" \ + --silent --location \ + "${GITHUB_API_URL}/repos/${GITHUB_REPO}/issues/${GITHUB_ISSUE}/comments" \ + >/dev/null + + retval=$? + rm "${restfile}" + return ${retval} +} + +## @description Print out the finished details to the Github PR +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function github_finalreport +{ + declare result=$1 + declare i + declare commentfile=${PATCH_DIR}/gitcommentfile.$$ + declare comment + + rm "${commentfile}" 2>/dev/null + + if [[ ${JENKINS} != "true" + || -z ${GITHUB_ISSUE} ]] ; then + return 0 + fi + + big_console_header "Adding comment to Github" + + add_footer_table "Console output" "${BUILD_URL}console" + + if [[ ${result} == 0 ]]; then + echo ":confetti_ball: **+1 overall**" >> "${commentfile}" + else + echo ":broken_heart: **-1 overall**" >> "${commentfile}" + fi + + printf "\n\n\n\n" >> "${commentfile}" + + i=0 + until [[ ${i} -eq ${#TP_HEADER[@]} ]]; do + printf "%s\n\n" "${TP_HEADER[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + + { + printf "\n\n" + echo "| Vote | Subsystem | Runtime | Comment |" + echo "|:----:|----------:|--------:|:--------|" + } >> "${commentfile}" + + i=0 + until [[ ${i} -eq ${#TP_VOTE_TABLE[@]} ]]; do + echo "${TP_VOTE_TABLE[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { + printf "\n\n" + echo "| Reason | Tests |" + echo "|-------:|:------|" + } >> "${commentfile}" + i=0 + until [[ ${i} -eq ${#TP_TEST_TABLE[@]} ]]; do + echo "${TP_TEST_TABLE[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + fi + + { + printf "\n\n" + echo "| Subsystem | Report/Notes |" + echo "|----------:|:-------------|" + } >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g") + printf "%s\n" "${comment}" >> "${commentfile}" + ((i=i+1)) + done + + printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" + + github_write_comment "${commentfile}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/gradle.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/gradle.sh b/precommit/test-patch.d/gradle.sh new file mode 100755 index 0000000..56da8d5 --- /dev/null +++ b/precommit/test-patch.d/gradle.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool gradle + +declare -a GRADLE_ARGS=() + +function gradle_usage +{ + echo "gradle specific:" + echo "--gradle-cmd=<cmd> The 'gradle' command to use (default 'gradle')" + echo "--gradlew-cmd=<cmd> The 'gradlew' command to use (default 'basedir/gradlew')" +} + +function gradle_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --gradle-cmd=*) + GRADLE=${i#*=} + ;; + --gradlew-cmd=*) + GRADLEW=${i#*=} + ;; + esac + done + + # if we requested offline, pass that to mvn + if [[ ${OFFLINE} == "true" ]]; then + GRADLE_ARGS=("${GRADLE_ARGS[@]}" --offline) + fi + + GRADLE=${GRADLE:-gradle} + GRADLEW=${GRADLEW:-"${BASEDIR}/gradlew"} +} + +function gradle_initialize +{ + if [[ "${BUILDTOOL}" = gradle ]]; then + # shellcheck disable=SC2034 + BUILDTOOLCWD=false + fi + + # we need to do this before docker kicks in + if [[ -e "${HOME}/.gradle" + && ! -d "${HOME}/.gradle" ]]; then + yetus_error "ERROR: ${HOME}/.gradle is not a directory." + return 1 + elif [[ ! -e "${HOME}/.gradle" ]]; then + yetus_debug "Creating ${HOME}/.gradle" + mkdir -p "${HOME}/.gradle" + fi +} + +function gradle_buildfile +{ + echo "gradlew" +} + +function gradle_executor +{ + echo "${GRADLEW}" "${GRADLE_ARGS[@]}" +} + +## @description Bootstrap gradle +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function gradle_precompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != gradle ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + big_console_header "${PATCH_BRANCH} gradle bootstrap" + else + big_console_header "Patch gradle bootstrap" + fi + + personality_modules "${repostatus}" gradleboot + + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/${repostatus}-gradle-bootstrap.txt" gradle -b bootstrap.gradle + popd >/dev/null + + modules_workers "${repostatus}" gradleboot + result=$? + modules_messages "${repostatus}" gradleboot true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function gradle_scalac_count_probs +{ + local warningfile=$1 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "^/.*.scala:[0-9]*:" "${warningfile}" | wc -l +} + +function gradle_javac_count_probs +{ + echo 0 +} + +function gradle_javadoc_count_probs +{ + echo 0 +} + +## @description Helper for check_patch_javadoc +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function gradle_scaladoc_count_probs +{ + local warningfile=$1 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "^\[ant:scaladoc\]" "${warningfile}" | wc -l +} + +function gradle_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + shift 2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + checkstyle) + modules_workers "${repostatus}" "${tst}" checkstyleMain checkstyleTest + ;; + compile) + modules_workers "${repostatus}" "${tst}" + ;; + distclean) + modules_workers "${repostatus}" clean + ;; + javadoc) + modules_workers "${repostatus}" "${tst}" javadoc + ;; + scaladoc) + modules_workers "${repostatus}" "${tst}" scaladoc + ;; + unit) + modules_workers "${repostatus}" "${tst}" test + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +function gradle_builtin_personality_modules +{ + local repostatus=$1 + local testtype=$2 + + local module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + for module in ${CHANGED_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} + done +} + +function gradle_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using builtin gradle personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/main/scripts + || ${filename} =~ src/test/scripts + ]]; then + yetus_debug "tests/shell: ${filename}" + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test cc + add_test unit + elif [[ ${filename} =~ \.scala$ ]]; then + add_test scalac + add_test scaladoc + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test javac + add_test javadoc + add_test unit + elif [[ ${filename} =~ src/main ]]; then + yetus_debug "tests/generic+units: ${filename}" + add_test compile + add_test unit + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +function gradle_docker_support +{ + echo "-v ${HOME}/.gradle:${HOME}/.gradle" > "${PATCH_DIR}/buildtool-docker-params.txt" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/java.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/java.sh b/precommit/test-patch.d/java.sh new file mode 100755 index 0000000..e1a73c7 --- /dev/null +++ b/precommit/test-patch.d/java.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type javac +add_test_type javadoc + +function javac_initialize +{ + local i + local jdkdir + local tmplist + + # if we are in pre-docker mode, don't do any of + # this work since it's all going to be wrong anyway + if [[ "${DOCKERSUPPORT}" = "true" ]]; then + return 0 + fi + + if [[ -z ${JAVA_HOME:-} ]]; then + case ${OSTYPE} in + Darwin) + if [[ -z "${JAVA_HOME}" ]]; then + if [[ -x /usr/libexec/java_home ]]; then + JAVA_HOME="$(/usr/libexec/java_home)" + export JAVA_HOME + else + export JAVA_HOME=/Library/Java/Home + fi + fi + ;; + *) + yetus_error "WARNING: JAVA_HOME not defined. Disabling java tests." + delete_test javac + delete_test javadoc + return 1 + ;; + esac + fi + + JAVA_HOME=$(cd -P -- "${JAVA_HOME}" >/dev/null && pwd -P) + + for i in ${JDK_DIR_LIST}; do + if [[ -d "${i}" ]]; then + jdkdir=$(cd -P -- "${i}" >/dev/null && pwd -P) + if [[ ${jdkdir} != "${JAVA_HOME}" ]]; then + tmplist="${tmplist} ${jdkdir}" + fi + else + yetus_error "WARNING: Cannot locate JDK directory ${i}: ignoring" + fi + done + + JDK_DIR_LIST="${tmplist} ${JAVA_HOME}" + JDK_DIR_LIST=${JDK_DIR_LIST/ } +} + +## @description Verify that ${JAVA_HOME} is defined +## @audience public +## @stability stable +## @replaceable no +## @return 1 - no JAVA_HOME +## @return 0 - JAVA_HOME defined +function javac_precheck +{ + declare javaversion + declare listofjdks + declare i + + start_clock + + if [[ -z ${JAVA_HOME:-} ]]; then + yetus_error "ERROR: JAVA_HOME is not defined." + add_vote_table -1 pre-patch "JAVA_HOME is not defined." + return 1 + fi + + javaversion=$(report_jvm_version "${JAVA_HOME}") + add_footer_table "Default Java" "${javaversion}" + + if [[ -n ${JDK_DIR_LIST} + && ${JDK_DIR_LIST} != "${JAVA_HOME}" ]]; then + for i in ${JDK_DIR_LIST}; do + javaversion=$(report_jvm_version "${i}") + listofjdks="${listofjdks} ${i}:${javaversion}" + done + add_footer_table "Multi-JDK versions" "${listofjdks}" + fi + return 0 +} + +function javac_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + yetus_debug "tests/javac: ${filename}" + add_test javac + add_test compile + fi +} + +function javadoc_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + yetus_debug "tests/javadoc: ${filename}" + add_test javadoc + fi +} + +## @description +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function javac_compile +{ + declare codebase=$1 + declare multijdkmode=$2 + + verify_needed_test javac + if [[ $? = 0 ]]; then + return 0 + fi + + if [[ ${codebase} = patch ]]; then + yetus_debug "javac: calling generic_postlog_compare compile javac ${multijdkmode}" + generic_postlog_compare compile javac "${multijdkmode}" + fi +} + +## @description Count and compare the number of JavaDoc warnings pre- and post- patch +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function javadoc_rebuild +{ + declare codebase=$1 + declare multijdkmode + + verify_multijdk_test javadoc + if [[ $? == 1 ]]; then + multijdkmode=true + fi + + if [[ "${codebase}" = branch ]]; then + generic_pre_handler javadoc "${multijdkmode}" + else + generic_post_handler javadoc javadoc "${multijdkmode}" true + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/jira.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/jira.sh b/precommit/test-patch.d/jira.sh new file mode 100755 index 0000000..e5fd550 --- /dev/null +++ b/precommit/test-patch.d/jira.sh @@ -0,0 +1,436 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this bug system handles JIRA. Personalities +# can override the following variables: + +# base JIRA URL +JIRA_URL=${JIRA_URL:-"https://issues.apache.org/jira"} + +# Issue regex to help identify the project +JIRA_ISSUE_RE='' + +# If the issue status is matched with this pattern, the attached patch is regarded as ready to be applied +JIRA_STATUS_RE='Patch Available' + +add_bugsystem jira + +function jira_usage +{ + echo "JIRA Options:" + echo "--jira-base-url=<url> The URL of the JIRA server (default:'${JIRA_URL}')" + echo "--jira-issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default: \'${JIRA_ISSUE_RE}\')" + echo "--jira-password=<pw> The password for the 'jira' command" + echo "--jira-status-re=<expr> Grep regular expression representing the issue status whose patch is applicable to the codebase (default: \'${JIRA_STATUS_RE}\')" + echo "--jira-user=<user> The user for the 'jira' command" +} + +function jira_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --jira-base-url=*) + JIRA_URL=${i#*=} + ;; + --jira-issue-re=*) + JIRA_ISSUE_RE=${i#*=} + ;; + --jira-password=*) + JIRA_PASSWD=${i#*=} + ;; + --jira-status-re=*) + JIRA_STATUS_RE=${i#*=} + ;; + --jira-user=*) + JIRA_USER=${i#*=} + ;; + esac + done +} + +## @description provides issue determination based upon the URL and more. +## @description WARNING: called from the github plugin! +function jira_determine_issue +{ + declare input=$1 + declare patchnamechunk + declare maybeissue + + if [[ -n ${JIRA_ISSUE} ]]; then + return 0 + fi + + # shellcheck disable=SC2016 + patchnamechunk=$(echo "${input}" | ${AWK} -F/ '{print $NF}') + + maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-) + + if [[ ${maybeissue} =~ ${JIRA_ISSUE_RE} ]]; then + # shellcheck disable=SC2034 + ISSUE=${maybeissue} + JIRA_ISSUE=${maybeissue} + add_footer_table "JIRA Issue" "${JIRA_ISSUE}" + return 0 + fi + + return 1 +} + +function jira_http_fetch +{ + declare input=$1 + declare output=$2 + + if [[ -n "${JIRA_USER}" + && -n "${JIRA_PASSWD}" ]]; then + ${CURL} --silent --fail \ + --user "${JIRA_USER}:${JIRA_PASSWD}" \ + --output "${output}" \ + --location \ + "${JIRA_URL}/${input}" + else + ${CURL} --silent --fail \ + --output "${output}" \ + --location \ + "${JIRA_URL}/${input}" + fi +} + +function jira_locate_patch +{ + declare input=$1 + declare fileloc=$2 + declare relativeurl + + yetus_debug "jira_locate_patch: trying ${JIRA_URL}/browse/${input}" + + if [[ "${OFFLINE}" == true ]]; then + yetus_debug "jira_locate_patch: offline, skipping" + return 1 + fi + + jira_http_fetch "browse/${input}" "${PATCH_DIR}/jira" + + if [[ $? != 0 ]]; then + yetus_debug "jira_locate_patch: not a JIRA." + return 1 + fi + + # if github is configured and we see what looks like a URL, + # send this to the github plugin to process. + if [[ -n "${GITHUB_BASE_URL}" + && $(${GREP} -c "${GITHUB_BASE_URL}"'[^ ]*patch' "${PATCH_DIR}/jira") != 0 ]]; then + jira_determine_issue "${input}" + echo "${input} appears to be a Github PR. Switching Modes." + github_jira_bridge "${fileloc}" + return $? + elif [[ $(${GREP} -c "${JIRA_STATUS_RE}" "${PATCH_DIR}/jira") == 0 ]]; then + if [[ ${JENKINS} == true ]]; then + yetus_error "ERROR: ${input} issue status is not matched with \"${JIRA_STATUS_RE}\"." + cleanup_and_exit 1 + else + yetus_error "WARNING: ${input} issue status is not matched with \"${JIRA_STATUS_RE}\"." + fi + fi + + #shellcheck disable=SC2016 + relativeurl=$(${AWK} 'match($0,"/secure/attachment/[0-9]*/[^\"]*"){print substr($0,RSTART,RLENGTH)}' "${PATCH_DIR}/jira" | + ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${SED} -e 's,[ ]*$,,g') + PATCHURL="${JIRA_URL}${relativeurl}" + if [[ ! ${PATCHURL} =~ \.patch$ ]]; then + guess_patch_file "${PATCH_DIR}/patch" + if [[ $? == 0 ]]; then + yetus_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. Proceeding, but issue/branch matching might go awry." + add_vote_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions." + fi + fi + echo "${input} patch is being downloaded at $(date) from" + echo "${PATCHURL}" + add_footer_table "JIRA Patch URL" "${PATCHURL}" + jira_http_fetch "${relativeurl}" "${fileloc}" + if [[ $? != 0 ]];then + yetus_error "ERROR: ${input}/${PATCHURL} could not be downloaded." + cleanup_and_exit 1 + fi + return 0 +} + +## @description Try to guess the branch being tested using a variety of heuristics +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success, with PATCH_BRANCH updated appropriately +function jira_determine_branch +{ + declare patchnamechunk + declare total + declare count + declare hinttype + + for hinttype in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do + if [[ -z "${hinttype}" ]]; then + continue + fi + + # If one of these matches the JIRA issue regex + # then we don't want it to trigger the branch + # detection since that's almost certainly not + # intended. In other words, if ISSUE-99 is the + # name of a branch, you want to test ISSUE-99 + # against master, not ISSUE-99's branch + if [[ ${hinttype} =~ ${JIRA_ISSUE_RE} ]]; then + continue + fi + + yetus_debug "Determine branch: starting with ${hinttype}" + patchnamechunk=$(echo "${hinttype}" \ + | ${SED} -e 's,.*/\(.*\)$,\1,' \ + -e 's,\.txt,.,' \ + -e 's,.patch,.,g' \ + -e 's,.diff,.,g' \ + -e 's,\.\.,.,g' \ + -e 's,\.$,,g' ) + + # ISSUE-branch-## + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-) + yetus_debug "Determine branch: ISSUE-branch-## = ${PATCH_BRANCH}" + if [[ -n "${PATCH_BRANCH}" ]]; then + verify_valid_branch "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return 0 + fi + fi + + # ISSUE-##[.##].branch + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d. ) + count="${PATCH_BRANCH//[^.]}" + total=${#count} + ((total = total + 3 )) + until [[ ${total} -lt 3 ]]; do + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3-${total} -d.) + yetus_debug "Determine branch: ISSUE[.##].branch = ${PATCH_BRANCH}" + ((total=total-1)) + if [[ -n "${PATCH_BRANCH}" ]]; then + verify_valid_branch "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return 0 + fi + fi + done + + # ISSUE.branch.## + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2- -d. ) + count="${PATCH_BRANCH//[^.]}" + total=${#count} + ((total = total + 3 )) + until [[ ${total} -lt 2 ]]; do + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2-${total} -d.) + yetus_debug "Determine branch: ISSUE.branch[.##] = ${PATCH_BRANCH}" + ((total=total-1)) + if [[ -n "${PATCH_BRANCH}" ]]; then + verify_valid_branch "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return 0 + fi + fi + done + + # ISSUE-branch.## + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1- -d. ) + count="${PATCH_BRANCH//[^.]}" + total=${#count} + ((total = total + 1 )) + until [[ ${total} -lt 1 ]]; do + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1-${total} -d. ) + yetus_debug "Determine branch: ISSUE-branch[.##] = ${PATCH_BRANCH}" + ((total=total-1)) + if [[ -n "${PATCH_BRANCH}" ]]; then + verify_valid_branch "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return 0 + fi + fi + done + done + + return 1 +} + +## @description Write the contents of a file to JIRA +## @params filename +## @stability stable +## @audience public +## @returns exit code from posting to jira +function jira_write_comment +{ + declare -r commentfile=${1} + declare retval=0 + + if [[ "${OFFLINE}" == true ]]; then + echo "JIRA Plugin: Running in offline, comment skipped." + return 0 + fi + + if [[ -n ${JIRA_PASSWD} + && -n ${JIRA_USER} ]]; then + + # RESTify the comment + { + echo "{\"body\":\"" + ${SED} -e 's,\\,\\\\,g' \ + -e 's,\",\\\",g' \ + -e 's,$,\\r\\n,g' "${commentfile}" \ + | tr -d '\n' + echo "\"}" + } > "${PATCH_DIR}/jiracomment.$$" + + ${CURL} -X POST \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -u "${JIRA_USER}:${JIRA_PASSWD}" \ + -d @"${PATCH_DIR}/jiracomment.$$" \ + --silent --location \ + "${JIRA_URL}/rest/api/2/issue/${JIRA_ISSUE}/comment" \ + >/dev/null + retval=$? + rm "${PATCH_DIR}/jiracomment.$$" + else + echo "JIRA Plugin: no credentials provided to write a comment." + fi + return ${retval} +} + +## @description Print out the finished details to the JIRA issue +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function jira_finalreport +{ + declare result=$1 + declare i + declare commentfile=${PATCH_DIR}/jiracommentfile + declare comment + declare vote + declare ourstring + declare ela + declare subs + declare color + declare comment + + rm "${commentfile}" 2>/dev/null + + if [[ ${JENKINS} == "false" + || ${OFFLINE} == true ]] ; then + return 0 + fi + + if [[ -z "${JIRA_ISSUE}" ]]; then + return 0 + fi + + big_console_header "Adding comment to JIRA" + + add_footer_table "Console output" "${BUILD_URL}console" + + if [[ ${result} == 0 ]]; then + echo "| (/) *{color:green}+1 overall{color}* |" >> "${commentfile}" + else + echo "| (x) *{color:red}-1 overall{color}* |" >> "${commentfile}" + fi + + echo "\\\\" >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_HEADER[@]} ]]; do + printf "%s\n" "${TP_HEADER[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + + echo "\\\\" >> "${commentfile}" + + echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}" + + i=0 + until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + comment=$(echo "${ourstring}" | cut -f5 -d\|) + + # summary line + if [[ -z ${vote} + && -n ${ela} ]]; then + color="black" + elif [[ -z ${vote} ]]; then + # keep same color + true + else + # new vote line + case ${vote} in + 1|"+1") + color="green" + ;; + -1) + color="red" + ;; + 0) + color="blue" + ;; + *) + color="black" + ;; + esac + fi + + printf "| {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} |\n" \ + "${color}" "${vote}" \ + "${color}" "${subs}" \ + "${color}" "${ela}" \ + "${color}" "${comment}" \ + >> "${commentfile}" + ((i=i+1)) + done + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + echo "|| Reason || Tests ||" >> "${commentfile}" + i=0 + until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do + printf "%s\n" "${TP_TEST_TABLE[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + fi + + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + echo "|| Subsystem || Report/Notes ||" >> "${commentfile}" + i=0 + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g") + printf "%s\n" "${comment}" >> "${commentfile}" + ((i=i+1)) + done + + printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" + + jira_write_comment "${commentfile}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/junit.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/junit.sh b/precommit/test-patch.d/junit.sh new file mode 100755 index 0000000..4393511 --- /dev/null +++ b/precommit/test-patch.d/junit.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_format junit + +JUNIT_TEST_TIMEOUTS="" +JUNIT_FAILED_TESTS="" + +function junit_process_tests +{ + # shellcheck disable=SC2034 + declare module=$1 + declare buildlogfile=$2 + declare result=0 + declare module_test_timeouts + declare module_failed_tests + + # shellcheck disable=SC2016 + module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${buildlogfile}") + if [[ -n "${module_test_timeouts}" ]] ; then + JUNIT_TEST_TIMEOUTS="${JUNIT_TEST_TIMEOUTS} ${module_test_timeouts}" + ((result=result+1)) + fi + + #shellcheck disable=SC2026,SC2038,SC2016 + module_failed_tests=$(find . -name 'TEST*.xml'\ + | xargs "${GREP}" -l -E "<failure|<error"\ + | ${AWK} -F/ '{sub("TEST-org.apache.",""); sub(".xml",""); print $NF}') + + if [[ -n "${module_failed_tests}" ]] ; then + JUNIT_FAILED_TESTS="${JUNIT_FAILED_TESTS} ${module_failed_tests}" + ((result=result+1)) + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +function junit_finalize_results +{ + declare jdk=$1 + + if [[ -n "${JUNIT_FAILED_TESTS}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "${jdk}Failed junit tests" ${JUNIT_FAILED_TESTS} + JUNIT_FAILED_TESTS="" + fi + if [[ -n "${JUNIT_TEST_TIMEOUTS}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "${jdk}Timed out junit tests" ${JUNIT_TEST_TIMEOUTS} + JUNIT_TEST_TIMEOUTS="" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/maven.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/maven.sh b/precommit/test-patch.d/maven.sh new file mode 100755 index 0000000..019b64d --- /dev/null +++ b/precommit/test-patch.d/maven.sh @@ -0,0 +1,440 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare -a MAVEN_ARGS=("--batch-mode") + +if [[ -z "${MAVEN_HOME:-}" ]]; then + MAVEN=mvn +else + MAVEN=${MAVEN_HOME}/bin/mvn +fi + +MAVEN_CUSTOM_REPOS=false +MAVEN_CUSTOM_REPOS_DIR="${HOME}/yetus-m2" + +add_test_type mvnsite +add_test_type mvneclipse +add_build_tool maven + +function maven_usage +{ + echo "maven specific:" + echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" + echo "--mvn-custom-repos Use per-project maven repos" + echo "--mvn-custom-repos-dir=dir Location of repos, default is \'${MAVEN_CUSTOM_REPOS_DIR}\'" + echo "--mvn-settings=file File to use for settings.xml" +} + +function maven_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --mvn-cmd=*) + MAVEN=${i#*=} + ;; + --mvn-custom-repos) + MAVEN_CUSTOM_REPOS=true + ;; + --mvn-custom-repos-dir=*) + MAVEN_CUSTOM_REPOS_DIR=${i#*=} + ;; + --mvn-settings=*) + MAVEN_SETTINGS=${i#*=} + if [[ -f ${MAVEN_SETTINGS} ]]; then + MAVEN_ARGS=("${MAVEN_ARGS[@]}" "--settings=${MAVEN_SETTINGS}") + else + yetus_error "WARNING: ${MAVEN_SETTINGS} not found. Ignorning." + fi + ;; + esac + done + + if [[ ${OFFLINE} == "true" ]]; then + MAVEN_ARGS=("${MAVEN_ARGS[@]}" --offline) + fi +} + +function maven_initialize +{ + # we need to do this before docker does it as root + + if [[ ! ${MAVEN_CUSTOM_REPOS_DIR} =~ ^/ ]]; then + yetus_error "ERROR: --mvn-custom-repos-dir must be an absolute path." + return 1 + fi + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + MAVEN_LOCAL_REPO="${MAVEN_CUSTOM_REPOS_DIR}" + if [[ -e "${MAVEN_CUSTOM_REPOS_DIR}" + && ! -d "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then + yetus_error "ERROR: ${MAVEN_CUSTOM_REPOS_DIR} is not a directory." + return 1 + elif [[ ! -d "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then + yetus_debug "Creating ${MAVEN_CUSTOM_REPOS_DIR}" + mkdir -p "${MAVEN_CUSTOM_REPOS_DIR}" + fi + fi + + if [[ -e "${HOME}/.m2" + && ! -d "${HOME}/.m2" ]]; then + yetus_error "ERROR: ${HOME}/.m2 is not a directory." + return 1 + elif [[ ! -e "${HOME}/.m2" ]]; then + yetus_debug "Creating ${HOME}/.m2" + mkdir -p "${HOME}/.m2" + fi +} + +function maven_precheck +{ + declare logfile="${PATCH_DIR}/mvnrepoclean.log" + declare line + + if [[ ! ${MAVEN_CUSTOM_REPOS_DIR} =~ ^/ ]]; then + yetus_error "ERROR: --mvn-custom-repos-dir must be an absolute path." + return 1 + fi + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + MAVEN_LOCAL_REPO="${MAVEN_CUSTOM_REPOS_DIR}/${PROJECT_NAME}-${PATCH_BRANCH}-${INSTANCE}" + if [[ -e "${MAVEN_LOCAL_REPO}" + && ! -d "${MAVEN_LOCAL_REPO}" ]]; then + yetus_error "ERROR: ${MAVEN_LOCAL_REPO} is not a directory." + return 1 + fi + + if [[ ! -d "${MAVEN_LOCAL_REPO}" ]]; then + yetus_debug "Creating ${MAVEN_LOCAL_REPO}" + mkdir -p "${MAVEN_LOCAL_REPO}" + if [[ $? -ne 0 ]]; then + yetus_error "ERROR: Unable to create ${MAVEN_LOCAL_REPO}" + return 1 + fi + fi + touch "${MAVEN_LOCAL_REPO}" + + # if we have a local settings.xml file, we copy it. + if [[ -f "${HOME}/.m2/settings.xml" ]]; then + cp -p "${HOME}/.m2/settings.xml" "${MAVEN_LOCAL_REPO}" + fi + MAVEN_ARGS=("${MAVEN_ARGS[@]}" "-Dmaven.repo.local=${MAVEN_LOCAL_REPO}") + + # let's do some cleanup while we're here + + find "${MAVEN_CUSTOM_REPOS_DIR}" \ + -name '*-*-*' \ + -type d \ + -mtime +30 \ + -maxdepth 1 \ + -print \ + > "${logfile}" + + while read -r line; do + echo "Removing old maven repo ${line}" + rm -rf "${line}" + done < "${logfile}" + fi +} + +function maven_buildfile +{ + echo "pom.xml" +} + +function maven_executor +{ + echo "${MAVEN}" "${MAVEN_ARGS[@]}" +} + +function mvnsite_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} = maven ]]; then + if [[ ${filename} =~ src/site ]]; then + yetus_debug "tests/mvnsite: ${filename}" + add_test mvnsite + fi + fi +} + +function maven_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + findbugs) + modules_workers "${repostatus}" findbugs test-compile findbugs:findbugs -DskipTests=true + ;; + compile) + modules_workers "${repostatus}" compile clean test-compile -DskipTests=true + ;; + distclean) + modules_workers "${repostatus}" distclean clean -DskipTests=true + ;; + javadoc) + modules_workers "${repostatus}" javadoc clean javadoc:javadoc -DskipTests=true + ;; + scaladoc) + modules_workers "${repostatus}" scaladoc clean scala:doc -DskipTests=true + ;; + unit) + modules_workers "${repostatus}" unit clean test -fae + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +function maven_javac_count_probs +{ + local warningfile=$1 + + #shellcheck disable=SC2016,SC2046 + ${GREP} '\[WARNING\]' "${warningfile}" | ${AWK} '{sum+=1} END {print sum}' +} + +function maven_scalac_count_probs +{ + echo 0 +} + +## @description Helper for check_patch_javadoc +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function maven_javadoc_count_probs +{ + local warningfile=$1 + + #shellcheck disable=SC2016,SC2046 + ${GREP} -E "^[0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}' +} + +function maven_builtin_personality_modules +{ + local repostatus=$1 + local testtype=$2 + + local module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + # this always makes sure the local repo has a fresh + # copy of everything per pom rules. + if [[ ${repostatus} == branch + && ${testtype} == mvninstall ]];then + personality_enqueue_module . + return + fi + + for module in ${CHANGED_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} + done +} + +function maven_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using builtin mvn personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/main/scripts + || ${filename} =~ src/test/scripts + ]]; then + yetus_debug "tests/shell: ${filename}" + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test cc + add_test unit + elif [[ ${filename} =~ \.scala$ + || ${filename} =~ src/scala ]]; then + add_test scalac + add_test scaladoc + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + || ${filename} =~ src/main + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test javac + add_test javadoc + add_test unit + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +## @description Confirm site pre-patch +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function mvnsite_postcompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != maven ]]; then + return 0 + fi + + verify_needed_test mvnsite + if [[ $? == 0 ]];then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + big_console_header "Pre-patch ${PATCH_BRANCH} maven site verification" + else + big_console_header "Patch maven site verification" + fi + + + personality_modules "${repostatus}" mvnsite + modules_workers "${repostatus}" mvnsite clean site site:stage + result=$? + modules_messages "${repostatus}" mvnsite true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description Make sure Maven's eclipse generation works. +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function mvneclipse_postcompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != maven ]]; then + return 0 + fi + + verify_needed_test javac + if [[ $? == 0 ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + big_console_header "Pre-patch ${PATCH_BRANCH} maven eclipse verification" + else + big_console_header "Patch maven eclipse verification" + fi + + personality_modules "${repostatus}" mvneclipse + modules_workers "${repostatus}" mvneclipse eclipse:clean eclipse:eclipse + result=$? + modules_messages "${repostatus}" mvneclipse true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description Verify mvn install works +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function maven_precompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != maven ]]; then + return 0 + fi + + verify_needed_test javadoc + result=$? + + verify_needed_test javac + ((result = result + $? )) + if [[ ${result} == 0 ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + big_console_header "Pre-patch ${PATCH_BRANCH} maven install" + else + big_console_header "Patch maven install" + fi + + personality_modules "${repostatus}" mvninstall + modules_workers "${repostatus}" mvninstall -fae clean install -DskipTests=true -Dmaven.javadoc.skip=true + result=$? + modules_messages "${repostatus}" mvninstall true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function maven_docker_support +{ + echo "-v ${HOME}/.m2:${HOME}/.m2" > "${PATCH_DIR}/buildtool-docker-params.txt" + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + echo "-v ${MAVEN_CUSTOM_REPOS_DIR}:${MAVEN_CUSTOM_REPOS_DIR}" \ + >> "${PATCH_DIR}/buildtool-docker-params.txt" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/nobuild.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/nobuild.sh b/precommit/test-patch.d/nobuild.sh new file mode 100644 index 0000000..74d35aa --- /dev/null +++ b/precommit/test-patch.d/nobuild.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool nobuild + +function nobuild_buildfile +{ + echo +} + +function nobuild_executor +{ + echo "true" +} + +function nobuild_modules_worker +{ + local status=$1 + local testtype=$2 + modules_workers "${status}" "${testtype}" +} + +function nobuild_builtin_personality_modules +{ + local status=$1 + local testtype=$2 + yetus_debug "built-in personality for no build system: ${status} ${testtype}" + + clear_personality_queue + for module in ${CHANGED_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} + done +} + +function nobuild_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using built-in no build system personality_file_tests." + yetus_debug " given file ${filename}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/perlcritic.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/perlcritic.sh b/precommit/test-patch.d/perlcritic.sh new file mode 100755 index 0000000..c55f01a --- /dev/null +++ b/precommit/test-patch.d/perlcritic.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type perlcritic + +PERLCRITIC_TIMER=0 + +PERLCRITIC=${PERLCRITIC:-$(which perlcritic 2>/dev/null)} + +function perlcritic_usage +{ + echo "Perl::Critic specific:" + echo "--perlcritic=<path> path to perlcritic executable" +} + +function perlcritic_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --perlcritic=*) + PERLCRITIC=${i#*=} + ;; + esac + done +} + +function perlcritic_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.p[lm]$ ]]; then + add_test perlcritic + fi +} + +function perlcritic_preapply +{ + local i + + verify_needed_test perlcritic + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "Perl::Critic plugin: prepatch" + + if [[ ! -x ${PERLCRITIC} ]]; then + yetus_error "${PERLCRITIC} does not exist." + return 0 + fi + + start_clock + + echo "Running perlcritic against modified perl scripts/modules." + pushd "${BASEDIR}" >/dev/null + for i in ${CHANGED_FILES}; do + if [[ ${i} =~ \.p[lm]$ && -f ${i} ]]; then + ${PERLCRITIC} -1 --verbose 1 "${i}" 2>/dev/null >> "${PATCH_DIR}/branch-perlcritic-result.txt" + fi + done + popd >/dev/null + # keep track of how much as elapsed for us already + PERLCRITIC_TIMER=$(stop_clock) + return 0 +} + +function perlcritic_postapply +{ + local i + local numPrepatch + local numPostpatch + local diffPostpatch + + verify_needed_test perlcritic + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "Perl::Critic plugin: postpatch" + + if [[ ! -x ${PERLCRITIC} ]]; then + yetus_error "${PERLCRITIC} is not available." + add_vote_table 0 perlcritic "Perl::Critic was not available." + return 0 + fi + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${PERLCRITIC_TIMER}" + + echo "Running perlcritic against modified perl scripts/modules." + # we re-check this in case one has been added + pushd "${BASEDIR}" >/dev/null + for i in ${CHANGED_FILES}; do + if [[ ${i} =~ \.p[lm]$ && -f ${i} ]]; then + ${PERLCRITIC} -1 --verbose 1 "${i}" 2>/dev/null >> "${PATCH_DIR}/patch-perlcritic-result.txt" + fi + done + popd >/dev/null + + PERLCRITIC_VERSION=$(${PERLCRITIC} --version 2>/dev/null) + add_footer_table perlcritic "v${PERLCRITIC_VERSION}" + + calcdiffs "${PATCH_DIR}/branch-perlcritic-result.txt" "${PATCH_DIR}/patch-perlcritic-result.txt" > "${PATCH_DIR}/diff-patch-perlcritic.txt" + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-perlcritic.txt" | ${AWK} '{print $1}') + + if [[ ${diffPostpatch} -gt 0 ]] ; then + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/branch-perlcritic-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patch-perlcritic-result.txt" | ${AWK} '{print $1}') + + add_vote_table -1 perlcritic "The applied patch generated "\ + "${diffPostpatch} new Perl::Critic issues (total was ${numPrepatch}, now ${numPostpatch})." + add_footer_table perlcritic "@@BASE@@/diff-patch-perlcritic.txt" + return 1 + fi + + add_vote_table +1 perlcritic "There were no new perlcritic issues." + return 0 +} + +function perlcritic_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + perlcritic_preapply + else + perlcritic_postapply + fi +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/pylint.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/pylint.sh b/precommit/test-patch.d/pylint.sh new file mode 100755 index 0000000..d5af000 --- /dev/null +++ b/precommit/test-patch.d/pylint.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type pylint + +PYLINT_TIMER=0 + +PYLINT=${PYLINT:-$(which pylint 2>/dev/null)} +PYLINT_OPTIONS=${PYLINT_OPTIONS:-} + +function pylint_usage +{ + echo "Pylint specific:" + echo "--pylint=<path> path to pylint executable" + echo "--pylint-options=<path> pylint options other than output-format and reports" +} + +function pylint_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --pylint=*) + PYLINT=${i#*=} + ;; + --pylint-options=*) + PYLINT_OPTIONS=${i#*=} + ;; + esac + done +} + +function pylint_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.py$ ]]; then + add_test pylint + fi +} + +function pylint_preapply +{ + local i + local count + local tmp=${PATCH_DIR}/pylint.$$.${RANDOM} + + verify_needed_test pylint + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "pylint plugin: prepatch" + + if [[ ! -x ${PYLINT} ]]; then + yetus_error "${PYLINT} does not exist." + return 0 + fi + + start_clock + + echo "Running pylint against modified python scripts." + pushd "${BASEDIR}" >/dev/null + for i in ${CHANGED_FILES}; do + if [[ ${i} =~ \.py$ && -f ${i} ]]; then + # shellcheck disable=SC2086 + eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>${tmp} | + ${AWK} '1<NR' >> "${PATCH_DIR}/branch-pylint-result.txt" + fi + # shellcheck disable=SC2016 + count=$(${GREP} -v "^No config file found" "${tmp}" | wc -l | ${AWK} '{print $1}') + if [[ ${count} -gt 0 ]]; then + add_footer_table pylint "prepatch stderr: ${tmp}" + return 1 + fi + done + rm "${tmp}" 2>/dev/null + popd >/dev/null + # keep track of how much as elapsed for us already + PYLINT_TIMER=$(stop_clock) + return 0 +} + +function pylint_postapply +{ + local i + local count + local numPrepatch + local numPostpatch + local diffPostpatch + local tmp=${PATCH_DIR}/pylint.$$.${RANDOM} + + verify_needed_test pylint + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "pylint plugin: postpatch" + + if [[ ! -x ${PYLINT} ]]; then + yetus_error "${PYLINT} is not available." + add_vote_table 0 pylint "Pylint was not available." + return 0 + fi + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${PYLINT_TIMER}" + + echo "Running pylint against modified python scripts." + # we re-check this in case one has been added + pushd "${BASEDIR}" >/dev/null + for i in ${CHANGED_FILES}; do + if [[ ${i} =~ \.py$ && -f ${i} ]]; then + # shellcheck disable=SC2086 + eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>${tmp} | + ${AWK} '1<NR' >> "${PATCH_DIR}/patch-pylint-result.txt" + fi + # shellcheck disable=SC2016 + count=$(${GREP} -v "^No config file found" "${tmp}" | wc -l | ${AWK} '{print $1}') + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 pylint "Something bad seems to have happened in running pylint. Please check pylint stderr files." + add_footer_table pylint "postpatch stderr: ${tmp}" + return 1 + fi + done + rm "${tmp}" 2>/dev/null + popd >/dev/null + + # shellcheck disable=SC2016 + PYLINT_VERSION=$(${PYLINT} --version 2>/dev/null | ${GREP} pylint | ${AWK} '{print $NF}') + add_footer_table pylint "v${PYLINT_VERSION%,}" + + calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" "${PATCH_DIR}/patch-pylint-result.txt" > "${PATCH_DIR}/diff-patch-pylint.txt" + diffPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/diff-patch-pylint.txt") + + if [[ ${diffPostpatch} -gt 0 ]] ; then + # shellcheck disable=SC2016 + numPrepatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/branch-pylint-result.txt") + + # shellcheck disable=SC2016 + numPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/patch-pylint-result.txt") + + add_vote_table -1 pylint "The applied patch generated "\ + "${diffPostpatch} new pylint issues (total was ${numPrepatch}, now ${numPostpatch})." + add_footer_table pylint "@@BASE@@/diff-patch-pylint.txt" + return 1 + fi + + add_vote_table +1 pylint "There were no new pylint issues." + return 0 +} + +function pylint_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + pylint_preapply + else + pylint_postapply + fi +}
