http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/maven.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/maven.sh b/precommit/src/main/shell/test-patch.d/maven.sh new file mode 100755 index 0000000..12fb239 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/maven.sh @@ -0,0 +1,761 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare -a MAVEN_ARGS + +if [[ -z "${MAVEN_HOME:-}" ]]; then + MAVEN=mvn +else + MAVEN=${MAVEN_HOME}/bin/mvn +fi + +MAVEN_CUSTOM_REPOS=false +MAVEN_CUSTOM_REPOS_DIR="@@@WORKSPACE@@@/yetus-m2" +MAVEN_DEPENDENCY_ORDER=true + +add_test_type mvnsite +add_build_tool maven + +## @description Add the given test type as requiring a mvn install during the branch phase +## @audience public +## @stability stable +## @replaceable yes +## @param test +function maven_add_install +{ + yetus_add_entry MAVEN_NEED_INSTALL "${1}" +} + +## @description Remove the given test type as requiring a mvn install +## @audience public +## @stability stable +## @replaceable yes +## @param test +function maven_delete_install +{ + yetus_delete_entry MAVEN_NEED_INSTALL "${1}" +} + +## @description replace the custom repo with either home or workspace if jenkins. +## @description is configured. this gets called in a few places since different +## @description circumstances dictate a few places where it may be needed. +## @audience private +## @stability evolving +function maven_ws_replace +{ + declare previous=${MAVEN_CUSTOM_REPOS_DIR} + + if [[ ${JENKINS} == true ]] && [[ -n "${WORKSPACE}" ]]; then + MAVEN_CUSTOM_REPOS_DIR=$(echo "${MAVEN_CUSTOM_REPOS_DIR}" | "${SED}" -e "s,@@@WORKSPACE@@@,${WORKSPACE},g" ) + else + MAVEN_CUSTOM_REPOS_DIR=$(echo "${MAVEN_CUSTOM_REPOS_DIR}" | "${SED}" -e "s,@@@WORKSPACE@@@,${HOME},g" ) + fi + if [[ "${previous}" != "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then + # put this in the array so that if docker is run, this is already resolved + USER_PARAMS=("${USER_PARAMS[@]}" "--mvn-custom-repos-dir=${MAVEN_CUSTOM_REPOS_DIR}") + fi +} + +function maven_usage +{ + maven_ws_replace + yetus_add_option "--mvn-cmd=<cmd>" "The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" + yetus_add_option "--mvn-custom-repos" "Use per-project maven repos" + yetus_add_option "--mvn-custom-repos-dir=dir" "Location of repos, default is '${MAVEN_CUSTOM_REPOS_DIR}'" + yetus_add_option "--mvn-deps-order=<bool>" "Disable maven's auto-dependency module ordering (Default: '${MAVEN_DEPENDENCY_ORDER}')" + yetus_add_option "--mvn-settings=file" "File to use for settings.xml" +} + +## @description parse maven build tool args +## @replaceable yes +## @audience public +## @stability stable +function maven_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --mvn-cmd=*) + MAVEN=${i#*=} + ;; + --mvn-custom-repos) + MAVEN_CUSTOM_REPOS=true + ;; + --mvn-custom-repos-dir=*) + MAVEN_CUSTOM_REPOS_DIR=${i#*=} + ;; + --mvn-deps-order=*) + MAVEN_DEPENDENCY_ORDER=${i#*=} + ;; + --mvn-settings=*) + MAVEN_SETTINGS=${i#*=} + if [[ -f ${MAVEN_SETTINGS} ]]; then + MAVEN_ARGS=("${MAVEN_ARGS[@]}" "--settings=${MAVEN_SETTINGS}") + else + yetus_error "WARNING: ${MAVEN_SETTINGS} not found. Ignoring." + fi + ;; + esac + done + + if [[ ${OFFLINE} == "true" ]]; then + MAVEN_ARGS=("${MAVEN_ARGS[@]}" --offline) + fi + + maven_ws_replace +} + +## @description initialize the maven build tool +## @replaceable yes +## @audience public +## @stability stable +function maven_initialize +{ + # we need to do this before docker does it as root + + maven_add_install compile + maven_add_install mvnsite + maven_add_install unit + + # Tell the reaper about the maven surefire plugin + reaper_add_name surefirebooter + + # we need to do this before docker does it as root + maven_ws_replace + + if [[ ! ${MAVEN_CUSTOM_REPOS_DIR} =~ ^/ ]]; then + yetus_error "ERROR: --mvn-custom-repos-dir must be an absolute path." + return 1 + fi + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + MAVEN_LOCAL_REPO="${MAVEN_CUSTOM_REPOS_DIR}" + if [[ -e "${MAVEN_CUSTOM_REPOS_DIR}" + && ! -d "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then + yetus_error "ERROR: ${MAVEN_CUSTOM_REPOS_DIR} is not a directory." + return 1 + elif [[ ! -d "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then + yetus_debug "Creating ${MAVEN_CUSTOM_REPOS_DIR}" + mkdir -p "${MAVEN_CUSTOM_REPOS_DIR}" + fi + fi + + if [[ -e "${HOME}/.m2" + && ! -d "${HOME}/.m2" ]]; then + yetus_error "ERROR: ${HOME}/.m2 is not a directory." + return 1 + elif [[ ! -e "${HOME}/.m2" ]]; then + yetus_debug "Creating ${HOME}/.m2" + mkdir -p "${HOME}/.m2" + fi +} + +## @audience private +## @stability stable +function mvnsite_precheck +{ + if ! verify_plugin_enabled 'maven'; then + yetus_error "ERROR: to run the mvnsite test you must ensure the 'maven' plugin is enabled." + return 1 + fi +} + +## @audience private +## @stability stable +function maven_precheck +{ + declare logfile="${PATCH_DIR}/mvnrepoclean.log" + declare line + declare maven_version + + if ! verify_plugin_enabled 'maven'; then + yetus_error "ERROR: you can't specify maven as the buildtool if you don't enable the plugin." + return 1 + fi + + if ! verify_command maven "${MAVEN}"; then + add_vote_table -1 maven "ERROR: maven was not available." + return 1 + fi + + if [[ ! ${MAVEN_CUSTOM_REPOS_DIR} =~ ^/ ]]; then + yetus_error "ERROR: --mvn-custom-repos-dir must be an absolute path." + return 1 + fi + + MAVEN_ARGS=("${MAVEN_ARGS[@]}" "--batch-mode") + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + MAVEN_LOCAL_REPO="${MAVEN_CUSTOM_REPOS_DIR}/${PROJECT_NAME}-${PATCH_BRANCH}-${BUILDMODE}-${INSTANCE}" + if [[ -e "${MAVEN_LOCAL_REPO}" + && ! -d "${MAVEN_LOCAL_REPO}" ]]; then + yetus_error "ERROR: ${MAVEN_LOCAL_REPO} is not a directory." + return 1 + fi + + if [[ ! -d "${MAVEN_LOCAL_REPO}" ]]; then + yetus_debug "Creating ${MAVEN_LOCAL_REPO}" + mkdir -p "${MAVEN_LOCAL_REPO}" + if [[ $? -ne 0 ]]; then + yetus_error "ERROR: Unable to create ${MAVEN_LOCAL_REPO}" + return 1 + fi + fi + touch "${MAVEN_LOCAL_REPO}" + + # if we have a local settings.xml file, we copy it. + if [[ -f "${HOME}/.m2/settings.xml" ]]; then + cp -p "${HOME}/.m2/settings.xml" "${MAVEN_LOCAL_REPO}" + fi + MAVEN_ARGS=("${MAVEN_ARGS[@]}" "-Dmaven.repo.local=${MAVEN_LOCAL_REPO}") + + # let's do some cleanup while we're here + + find "${MAVEN_CUSTOM_REPOS_DIR}" \ + -name '*-*-*' \ + -type d \ + -mtime +30 \ + -maxdepth 1 \ + -print \ + > "${logfile}" + + while read -r line; do + echo "Removing old maven repo ${line}" + rm -rf "${line}" + done < "${logfile}" + fi + + # finally let folks know what version they'll be dealing with. + maven_version=$(${MAVEN} --offline --version 2>/dev/null | head -n 1 2>/dev/null) + add_footer_table maven "version: ${maven_version}" +} + +function maven_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ pom\.xml$ ]]; then + yetus_debug "tests/compile: ${filename}" + add_test compile + fi +} + +function maven_buildfile +{ + echo "pom.xml" +} + +function maven_executor +{ + echo "${MAVEN}" "${MAVEN_ARGS[@]}" +} + +function mvnsite_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} = maven ]]; then + if [[ ${filename} =~ src/site ]]; then + yetus_debug "tests/mvnsite: ${filename}" + add_test mvnsite + fi + fi +} + +## @description maven version of the modules_worker routine +## @audience public +## @stability stable +function maven_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + declare maven_unit_test_filter + + maven_unit_test_filter="$(maven_unit_test_filter)" + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + findbugs) + modules_workers "${repostatus}" findbugs test-compile findbugs:findbugs -DskipTests=true + ;; + compile) + modules_workers "${repostatus}" compile clean test-compile -DskipTests=true + ;; + distclean) + modules_workers "${repostatus}" distclean clean -DskipTests=true + ;; + javadoc) + modules_workers "${repostatus}" javadoc clean javadoc:javadoc -DskipTests=true + ;; + scaladoc) + modules_workers "${repostatus}" scaladoc clean scala:doc -DskipTests=true + ;; + unit) + if [[ -n "${maven_unit_test_filter}" ]]; then + modules_workers "${repostatus}" unit clean test -fae "${maven_unit_test_filter}" + else + modules_workers "${repostatus}" unit clean test -fae + fi + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +function maven_javac_logfilter +{ + declare input=$1 + declare output=$2 + + ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" > "${output}" +} + +## @description Helper for check_patch_javadoc +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function maven_javadoc_logfilter +{ + declare input=$1 + declare output=$2 + + ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" > "${output}" +} + +## @description handle diffing maven javac errors +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function maven_javac_calcdiffs +{ + declare orig=$1 + declare new=$2 + declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} + declare j + + # first, strip :[line + # this keeps file,column in an attempt to increase + # accuracy in case of multiple, repeated errors + # since the column number shouldn't change + # if the line of code hasn't been touched + # shellcheck disable=SC2016 + ${SED} -e 's#:\[[0-9]*,#:#' "${orig}" > "${tmp}.branch" + # shellcheck disable=SC2016 + ${SED} -e 's#:\[[0-9]*,#:#' "${new}" > "${tmp}.patch" + + # compare the errors, generating a string of line + # numbers. Sorry portability: GNU diff makes this too easy + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${new}" | tail -1 + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + +## @description handle diffing maven javadoc errors +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function maven_javadoc_calcdiffs +{ + declare orig=$1 + declare new=$2 + declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} + declare j + + # can't use the generic handler for this because of the + # [WARNING], etc headers. + # strip :linenum from the output, keeping the filename + # shellcheck disable=SC2016 + ${SED} -e 's#:[0-9]*:#:#' "${orig}" > "${tmp}.branch" + # shellcheck disable=SC2016 + ${SED} -e 's#:[0-9]*:#:#' "${new}" > "${tmp}.patch" + + # compare the errors, generating a string of line + # numbers. Sorry portability: GNU diff makes this too easy + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${new}" | tail -1 + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + +function maven_builtin_personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + + declare module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + # this always makes sure the local repo has a fresh + # copy of everything per pom rules. + if [[ ${repostatus} == branch + && ${testtype} == mvninstall ]] || + [[ "${BUILDMODE}" = full ]];then + personality_enqueue_module "${CHANGED_UNION_MODULES}" + return + fi + + for module in "${CHANGED_MODULES[@]}"; do + personality_enqueue_module "${module}" + done +} + +function maven_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using builtin mvn personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/main/scripts + || ${filename} =~ src/test/scripts + ]]; then + yetus_debug "tests/shell: ${filename}" + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test cc + add_test unit + elif [[ ${filename} =~ \.scala$ + || ${filename} =~ src/scala ]]; then + add_test scalac + add_test scaladoc + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + || ${filename} =~ src/main + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test javac + add_test javadoc + add_test unit + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +## @description Maven unit test filter file string +## @audience private +## @stability evolving +## @replaceable no +## @return string +function maven_unit_test_filter() +{ + declare filtered + + if [[ ! -z "${UNIT_TEST_FILTER_FILE}" ]]; then + while read -r line || [[ -n "${line}" ]]; do + if [[ -z $line ]]; then + continue + fi + + filtered="${filtered}${line}," + done < "${UNIT_TEST_FILTER_FILE}" + fi + + if [[ -z "${filtered}" ]]; then + printf "%s" "" + else + printf "%s" "-Dtest=${filtered%,}" + fi +} + +## @description Confirm site pre-patch +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function mvnsite_postcompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != maven ]]; then + return 0 + fi + + if ! verify_needed_test mvnsite; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + big_console_header "maven site verification: ${PATCH_BRANCH}" + else + big_console_header "maven site verification: ${BUILDMODE}" + fi + + personality_modules "${repostatus}" mvnsite + modules_workers "${repostatus}" mvnsite clean site site:stage + result=$? + modules_messages "${repostatus}" mvnsite true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description maven precompile phase +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function maven_precompile +{ + declare repostatus=$1 + declare result=0 + declare need=false + + if [[ ${BUILDTOOL} != maven ]]; then + return 0 + fi + + # not everything needs a maven install + # but quite a few do ... + # shellcheck disable=SC2086 + for index in ${MAVEN_NEED_INSTALL}; do + if verify_needed_test "${index}"; then + need=true + fi + done + + if [[ "${need}" = false ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + big_console_header "maven install: ${PATCH_BRANCH}" + else + big_console_header "maven install: ${BUILDMODE}" + fi + + personality_modules "${repostatus}" mvninstall + modules_workers "${repostatus}" mvninstall -fae \ + clean install \ + -DskipTests=true -Dmaven.javadoc.skip=true \ + -Dcheckstyle.skip=true -Dfindbugs.skip=true + result=$? + modules_messages "${repostatus}" mvninstall true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +function maven_docker_support +{ + DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${HOME}/.m2:/home/${USER_NAME}/.m2") + + if [[ ${MAVEN_CUSTOM_REPOS} = true ]]; then + DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${MAVEN_CUSTOM_REPOS_DIR}:${MAVEN_CUSTOM_REPOS_DIR}") + fi +} + +## @description worker for maven reordering. MAVEN_DEP_LOG is set to the log file name +## @audience private +## @stability evolving +## @replaceable no +## @param repostatus +## @return 0 = success +## @return 1 = failure +function maven_reorder_module_process +{ + declare repostatus=$1 + declare module + declare line + declare indexm + declare indexn + declare -a newlist + declare fn + declare needroot=false + declare found + declare ret + + for module in "${CHANGED_MODULES[@]}"; do + if [[ "${module}" = \. ]]; then + needroot=true + fi + done + + fn=$(module_file_fragment "${CHANGED_UNION_MODULES}") + pushd "${BASEDIR}/${CHANGED_UNION_MODULES}" >/dev/null + + # get the module directory list in the correct order based on maven dependencies + # shellcheck disable=SC2046 + echo_and_redirect "${PATCH_DIR}/maven-${repostatus}-dirlist-${fn}.txt" \ + $("${BUILDTOOL}_executor") "-fae" "-q" "exec:exec" "-Dexec.executable=pwd" "-Dexec.args=''" + MAVEN_DEP_LOG="maven-${repostatus}-dirlist-${fn}.txt" + ret=$? + + while read -r line; do + for indexm in "${CHANGED_MODULES[@]}"; do + if [[ ${line} == "${BASEDIR}/${indexm}" ]]; then + yetus_debug "mrm: placing ${indexm} from dir: ${line}" + newlist=("${newlist[@]}" "${indexm}") + break + fi + done + done < "${PATCH_DIR}/maven-${repostatus}-dirlist-${fn}.txt" + popd >/dev/null + + if [[ "${needroot}" = true ]]; then + newlist=("${newlist[@]}" ".") + fi + + indexm="${#CHANGED_MODULES[@]}" + indexn="${#newlist[@]}" + + if [[ ${indexm} -ne ${indexn} ]]; then + yetus_debug "mrm: Missed a module" + for indexm in "${CHANGED_MODULES[@]}"; do + found=false + for indexn in "${newlist[@]}"; do + if [[ "${indexn}" = "${indexm}" ]]; then + found=true + break + fi + done + if [[ ${found} = false ]]; then + yetus_debug "mrm: missed ${indexm}" + newlist=("${newlist[@]}" "${indexm}") + fi + done + fi + + CHANGED_MODULES=("${newlist[@]}") + return "${ret}" +} + +## @description take a stab at reordering modules based upon +## @description maven dependency order +## @audience private +## @stability evolving +## @replaceable no +## @param repostatus +## @param module +function maven_reorder_modules +{ + declare repostatus=$1 + declare index + declare ret + + if [[ "${MAVEN_DEPENDENCY_ORDER}" != "true" ]]; then + return + fi + + # don't bother if there is only one + index="${#CHANGED_MODULES[@]}" + if [[ ${index} -eq 1 ]]; then + return + fi + + big_console_header "Determining Maven Dependency Order (downloading dependencies in the process)" + + start_clock + + maven_reorder_module_process "${repostatus}" + ret=$? + + yetus_debug "Maven: finish re-ordering modules" + yetus_debug "Finished list: ${CHANGED_MODULES[*]}" + + # build some utility module lists for maven modules + for index in "${CHANGED_MODULES[@]}"; do + if [[ -d "${index}/src" ]]; then + MAVEN_SRC_MODULES=("${MAVEN_SRC_MODULES[@]}" "${index}") + if [[ -d "${index}/src/test" ]]; then + MAVEN_SRCTEST_MODULES=("${MAVEN_SRCTEST_MODULES[@]}" "${index}") + fi + fi + done + + if [[ "${BUILDMODE}" = patch ]]; then + if [[ ${ret} == 0 ]]; then + add_vote_table 0 mvndep "Maven dependency ordering for ${repostatus}" + else + add_vote_table -1 mvndep "Maven dependency ordering for ${repostatus}" + add_footer_table mvndep "${MAVEN_DEP_LOG}" + fi + else + if [[ ${ret} == 0 ]]; then + add_vote_table 0 mvndep "Maven dependency ordering" + else + add_vote_table -1 mvndep "Maven dependency ordering" + add_footer_table mvndep "${MAVEN_DEP_LOG}" + fi + fi + + echo "Elapsed: $(clock_display $(stop_clock))" +}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/nobuild.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/nobuild.sh b/precommit/src/main/shell/test-patch.d/nobuild.sh new file mode 100755 index 0000000..cf5a2ad --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/nobuild.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool nobuild + +function nobuild_buildfile +{ + echo +} + +function nobuild_executor +{ + echo "true" +} + +function nobuild_modules_worker +{ + local status=$1 + local testtype=$2 + modules_workers "${status}" "${testtype}" +} + +function nobuild_builtin_personality_modules +{ + local status=$1 + local testtype=$2 + yetus_debug "built-in personality for no build system: ${status} ${testtype}" + + clear_personality_queue + for module in "${CHANGED_MODULES[@]}"; do + personality_enqueue_module "${module}" + done +} + +function nobuild_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using built-in no build system personality_file_tests." + yetus_debug " given file ${filename}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/pathlen.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/pathlen.sh b/precommit/src/main/shell/test-patch.d/pathlen.sh new file mode 100755 index 0000000..a013adb --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/pathlen.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type pathlen + +PATHLEN_SIZE=240 + +## @description pathlen usage hook +## @audience private +## @stability evolving +## @replaceable no +function pathlen_usage +{ + yetus_add_option "--pathlen-size=<int>" "reject patches with this size of paths (default: ${PATHLEN_SIZE}" + +} + +## @description pathlen parse args hook +## @audience private +## @stability evolving +## @replaceable no +function pathlen_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --pathlen-size=*) + PATHLEN_SIZE="${i#*=}" + ;; + esac + done +} + +## @description helper function to count long pathnames +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function pathlen_generic +{ + declare size + declare i + declare msg + declare counter + + counter=0 + + if [[ "${BUILDMODE}" = full ]]; then + msg="source tree" + else + msg="patch" + fi + + for i in "${CHANGED_FILES[@]}"; do + size=${#i} + if [[ ${size} -gt ${PATHLEN_SIZE} ]]; then + ((counter = counter + 1 )) + echo "${i}" >> "${PATCH_DIR}/pathlen.txt" + fi + done + + # shellcheck disable=SC2016 + echo "${counter} files in the ${msg} with paths longer that ${PATHLEN_SIZE}." + if [[ ${counter} -gt 0 ]] ; then + add_vote_table -1 pathlen \ + "${BUILDMODEMSG} appears to contain ${counter} files with names longer than ${PATHLEN_SIZE}" + add_footer_table pathlen "@@BASE@@/pathlen.txt" + return 1 + fi + return 0 +} + +## @description Check the current patchfile for @pathlen tags +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +## @param patchfile +function pathlen_patchfile +{ + if [[ "${BUILDMODE}" != patch ]]; then + return + fi + + big_console_header "Checking for long paths: ${BUILDMODE}" + + start_clock + + pathlen_generic +} + + +## @description Check the current directory for @pathlen tags +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function pathlen_postcompile +{ + if [[ "${BUILDMODE}" != full ]]; then + return + fi + + big_console_header "Checking for long paths: ${BUILDMODE}" + + start_clock + + pathlen_generic +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/perlcritic.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/perlcritic.sh b/precommit/src/main/shell/test-patch.d/perlcritic.sh new file mode 100755 index 0000000..291e94e --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/perlcritic.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type perlcritic + +PERLCRITIC_TIMER=0 + +PERLCRITIC=${PERLCRITIC:-$(which perlcritic 2>/dev/null)} + +function perlcritic_usage +{ + yetus_add_option "--perlcritic=<path>" "path to perlcritic executable" +} + +function perlcritic_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --perlcritic=*) + PERLCRITIC=${i#*=} + ;; + esac + done +} + +function perlcritic_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.p[lm]$ ]]; then + add_test perlcritic + fi +} + +function perlcritic_precheck +{ + if ! verify_command "Perl::Critic" "${PERLCRITIC}"; then + add_vote_table 0 perlcritic "Perl::Critic was not available." + delete_test perlcritic + fi +} + + +function perlcritic_preapply +{ + local i + + if ! verify_needed_test perlcritic; then + return 0 + fi + + big_console_header "Perl::Critic plugin: ${PATCH_BRANCH}" + + start_clock + + echo "Running perlcritic against identified perl scripts/modules." + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.p[lm]$ && -f ${i} ]]; then + ${PERLCRITIC} -1 --verbose 1 "${i}" 2>/dev/null >> "${PATCH_DIR}/branch-perlcritic-result.txt" + fi + done + popd >/dev/null + # keep track of how much as elapsed for us already + PERLCRITIC_TIMER=$(stop_clock) + return 0 +} + +## @description Wrapper to call column_calcdiffs +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function perlcritic_calcdiffs +{ + column_calcdiffs "$@" +} + +function perlcritic_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test perlcritic; then + return 0 + fi + + big_console_header "Perl::Critic plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${PERLCRITIC_TIMER}" + + echo "Running perlcritic against identified perl scripts/modules." + # we re-check this in case one has been added + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.p[lm]$ && -f ${i} ]]; then + ${PERLCRITIC} -1 --verbose 1 "${i}" 2>/dev/null >> "${PATCH_DIR}/patch-perlcritic-result.txt" + fi + done + popd >/dev/null + + PERLCRITIC_VERSION=$(${PERLCRITIC} --version 2>/dev/null) + add_footer_table perlcritic "v${PERLCRITIC_VERSION}" + + calcdiffs \ + "${PATCH_DIR}/branch-perlcritic-result.txt" \ + "${PATCH_DIR}/patch-perlcritic-result.txt" \ + perlcritic \ + > "${PATCH_DIR}/diff-patch-perlcritic.txt" + + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/branch-perlcritic-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patch-perlcritic-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-perlcritic.txt" | ${AWK} '{print $1}') + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]]; then + add_vote_table -1 perlcritic "${BUILDMODEMSG} ${statstring}" + add_footer_table perlcritic "@@BASE@@/diff-patch-perlcritic.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 perlcritic "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 perlcritic "There were no new perlcritic issues." + return 0 +} + +function perlcritic_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + perlcritic_preapply + else + perlcritic_postapply + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/pylint.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/pylint.sh b/precommit/src/main/shell/test-patch.d/pylint.sh new file mode 100755 index 0000000..facba6c --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/pylint.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type pylint + +PYLINT_TIMER=0 + +PYLINT=${PYLINT:-$(which pylint 2>/dev/null)} +PYLINT_OPTIONS=${PYLINT_OPTIONS:-} + +function pylint_usage +{ + yetus_add_option "--pylint=<path>" "path to pylint executable" + yetus_add_option "--pylint-options=<path>" "pylint options other than output-format and reports" +} + +function pylint_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --pylint=*) + PYLINT=${i#*=} + ;; + --pylint-options=*) + PYLINT_OPTIONS=${i#*=} + ;; + esac + done +} + +function pylint_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.py$ ]]; then + add_test pylint + fi +} + +function pylint_precheck +{ + if ! verify_command "Pylint" "${PYLINT}"; then + add_vote_table 0 pylint "Pylint was not available." + delete_test pylint + fi +} + + +function pylint_preapply +{ + local i + local count + local pylintStderr=branch-pylint-stderr.txt + + if ! verify_needed_test pylint; then + return 0 + fi + + big_console_header "pylint plugin: ${PATCH_BRANCH}" + + start_clock + + echo "Running pylint against identified python scripts." + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.py$ && -f ${i} ]]; then + # shellcheck disable=SC2086 + eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' --reports=n ${i}" \ + 2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1<NR' >> "${PATCH_DIR}/branch-pylint-result.txt" + fi + done + if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then + count=$(${GREP} -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}") + if [[ ${count} -gt 0 ]]; then + add_footer_table pylint "${PATCH_BRANCH} stderr: @@BASE@@/${pylintStderr}" + return 1 + fi + fi + rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null + popd >/dev/null + # keep track of how much as elapsed for us already + PYLINT_TIMER=$(stop_clock) + return 0 +} + +function pylint_postapply +{ + declare i + declare count + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare pylintStderr=patch-pylint-stderr.txt + declare fixedpatch + declare statstring + + if ! verify_needed_test pylint; then + return 0 + fi + + big_console_header "pylint plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${PYLINT_TIMER}" + + echo "Running pylint against identified python scripts." + # we re-check this in case one has been added + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.py$ && -f ${i} ]]; then + # shellcheck disable=SC2086 + eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' --reports=n ${i}" \ + 2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1<NR' >> "${PATCH_DIR}/patch-pylint-result.txt" + fi + done + if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then + count=$(${GREP} -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}") + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 pylint "Something bad seems to have happened in running pylint. Please check pylint stderr files." + add_footer_table pylint "${BUILDMODEMSG} stderr: @@BASE@@/${pylintStderr}" + return 1 + fi + fi + rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null + popd >/dev/null + + # shellcheck disable=SC2016 + PYLINT_VERSION=$(${PYLINT} --version 2>/dev/null | ${GREP} pylint | ${AWK} '{print $NF}') + add_footer_table pylint "v${PYLINT_VERSION%,}" + + calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" \ + "${PATCH_DIR}/patch-pylint-result.txt" \ + pylint > "${PATCH_DIR}/diff-patch-pylint.txt" + numPrepatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/branch-pylint-result.txt") + numPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/patch-pylint-result.txt") + # Exclude Pylint messages from the information category to avoid false positives (see YETUS-309). + diffPostpatch=$(${GREP} -c "^.*:.*: \[[^I].*\] " "${PATCH_DIR}/diff-patch-pylint.txt") + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_vote_table -1 pylint "${BUILDMODEMSG} ${statstring}" + add_footer_table pylint "@@BASE@@/diff-patch-pylint.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 pylint "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 pylint "There were no new pylint issues." + return 0 +} + +function pylint_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + pylint_preapply + else + pylint_postapply + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/rubocop.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/rubocop.sh b/precommit/src/main/shell/test-patch.d/rubocop.sh new file mode 100755 index 0000000..828bea1 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/rubocop.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type rubocop + +RUBOCOP_TIMER=0 + +RUBOCOP=${RUBOCOP:-$(which rubocop 2>/dev/null)} + +function rubocop_usage +{ + yetus_add_option "--rubocop=<path>" "path to rubocop executable" +} + +function rubocop_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --rubocop=*) + RUBOCOP=${i#*=} + ;; + esac + done +} + +function rubocop_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.rb$ ]]; then + add_test rubocop + fi +} + +function rubocop_precheck +{ + if ! verify_command rubocop "${RUBOCOP}"; then + add_vote_table 0 rubocop "rubocop was not available." + delete_test rubocop + fi +} + + +function rubocop_preapply +{ + local i + + if ! verify_needed_test rubocop; then + return 0 + fi + + big_console_header "rubocop plugin: ${PATCH_BRANCH}" + + start_clock + + echo "Running rubocop against identified ruby scripts." + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.rb$ && -f ${i} ]]; then + ${RUBOCOP} -f e "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/branch-rubocop-result.txt" + fi + done + popd >/dev/null + # keep track of how much as elapsed for us already + RUBOCOP_TIMER=$(stop_clock) + return 0 +} + +## @description Wrapper to call column_calcdiffs +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function rubocop_calcdiffs +{ + column_calcdiffs "$@" +} + +function rubocop_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test rubocop; then + return 0 + fi + + big_console_header "rubocop plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${RUBOCOP_TIMER}" + + echo "Running rubocop against identified ruby scripts." + # we re-check this in case one has been added + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.rb$ && -f ${i} ]]; then + ${RUBOCOP} -f e "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/patch-rubocop-result.txt" + fi + done + popd >/dev/null + + # shellcheck disable=SC2016 + RUBOCOP_VERSION=$(${RUBOCOP} -v | ${AWK} '{print $NF}') + add_footer_table rubocop "v${RUBOCOP_VERSION}" + + calcdiffs \ + "${PATCH_DIR}/branch-rubocop-result.txt" \ + "${PATCH_DIR}/patch-rubocop-result.txt" \ + rubocop \ + > "${PATCH_DIR}/diff-patch-rubocop.txt" + diffPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-rubocop.txt") + + # shellcheck disable=SC2016 + numPrepatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-rubocop-result.txt") + + # shellcheck disable=SC2016 + numPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-rubocop-result.txt") + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_vote_table -1 rubocop "${BUILDMODEMSG} ${statstring}" + add_footer_table rubocop "@@BASE@@/diff-patch-rubocop.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 rubocop "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 rubocop "There were no new rubocop issues." + return 0 +} + +function rubocop_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + rubocop_preapply + else + rubocop_postapply + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/ruby-lint.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/ruby-lint.sh b/precommit/src/main/shell/test-patch.d/ruby-lint.sh new file mode 100755 index 0000000..78be7b6 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/ruby-lint.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type ruby_lint + +RUBY_LINT_TIMER=0 + +RUBY_LINT=${RUBY_LINT:-$(which ruby-lint 2>/dev/null)} + +function ruby_lint_usage +{ + yetus_add_option "--ruby-lint=<path>" "path to ruby-lint executable" +} + +function ruby_lint_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --ruby-lint=*) + RUBY_LINT=${i#*=} + ;; + esac + done +} + +function ruby_lint_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.rb$ ]]; then + add_test ruby_lint + fi +} + +function ruby_lint_precheck +{ + if ! verify_command "Ruby-lint" "${RUBY_LINT}"; then + add_vote_table 0 ruby-lint "Ruby-lint was not available." + delete_test ruby_lint + fi +} + +function ruby_lint_preapply +{ + local i + + if ! verify_needed_test ruby_lint; then + return 0 + fi + + big_console_header "ruby-lint plugin: ${PATCH_BRANCH}" + + start_clock + + echo "Running ruby-lint against identified ruby scripts." + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.rb$ && -f ${i} ]]; then + ${RUBY_LINT} -p syntastic "${i}" | sort -t : -k 1,1 -k 3,3n -k 4,4n >> "${PATCH_DIR}/branch-ruby-lint-result.txt" + fi + done + popd >/dev/null + # keep track of how much as elapsed for us already + RUBY_LINT_TIMER=$(stop_clock) + return 0 +} + +## @description Calculate the differences between the specified files +## @description using columns and output it to stdout +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function ruby_lint_calcdiffs +{ + declare orig=$1 + declare new=$2 + declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} + declare j + + # first, strip filenames:line: + # this keeps column: in an attempt to increase + # accuracy in case of multiple, repeated errors + # since the column number shouldn't change + # if the line of code hasn't been touched + # shellcheck disable=SC2016 + cut -f4- -d: "${orig}" > "${tmp}.branch" + # shellcheck disable=SC2016 + cut -f4- -d: "${new}" > "${tmp}.patch" + + # compare the errors, generating a string of line + # numbers. Sorry portability: GNU diff makes this too easy + ${DIFF} --unchanged-line-format="" \ + --old-line-format="" \ + --new-line-format="%dn " \ + "${tmp}.branch" \ + "${tmp}.patch" > "${tmp}.lined" + + # now, pull out those lines of the raw output + # shellcheck disable=SC2013 + for j in $(cat "${tmp}.lined"); do + # shellcheck disable=SC2086 + head -${j} "${new}" | tail -1 + done + + rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null +} + +function ruby_lint_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test ruby_lint; then + return 0 + fi + + big_console_header "ruby-lint plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${RUBY_LINT_TIMER}" + + echo "Running ruby-lint against identified ruby scripts." + # we re-check this in case one has been added + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.rb$ && -f ${i} ]]; then + ${RUBY_LINT} -p syntastic "${i}" | sort -t : -k 1,1 -k 3,3n -k 4,4n >> "${PATCH_DIR}/patch-ruby-lint-result.txt" + fi + done + popd >/dev/null + + # shellcheck disable=SC2016 + RUBY_LINT_VERSION=$(${RUBY_LINT} -v | ${AWK} '{print $2}') + add_footer_table ruby-lint "${RUBY_LINT_VERSION}" + + calcdiffs \ + "${PATCH_DIR}/branch-ruby-lint-result.txt" \ + "${PATCH_DIR}/patch-ruby-lint-result.txt" \ + ruby_lint \ + > "${PATCH_DIR}/diff-patch-ruby-lint.txt" + diffPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-ruby-lint.txt") + + # shellcheck disable=SC2016 + numPrepatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-ruby-lint-result.txt") + + # shellcheck disable=SC2016 + numPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-ruby-lint-result.txt") + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_vote_table -1 ruby-lint "${BUILDMODEMSG} ${statstring}" + add_footer_table ruby-lint "@@BASE@@/diff-patch-ruby-lint.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 ruby-lint "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 ruby-lint "There were no new ruby-lint issues." + return 0 +} + +function ruby_lint_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + ruby_lint_preapply + else + ruby_lint_postapply + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/scala.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/scala.sh b/precommit/src/main/shell/test-patch.d/scala.sh new file mode 100755 index 0000000..f2b599a --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/scala.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type scalac +add_test_type scaladoc + +SCALA_INITIALIZED=false + +function scalac_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.scala$ ]]; then + yetus_debug "tests/scalac: ${filename}" + add_test scalac + add_test compile + fi +} + +function scaladoc_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.scala$ ]]; then + yetus_debug "tests/scaladoc: ${filename}" + add_test scaladoc + fi +} + +## @description initialize the scala plug-in +## @audience private +## @stability evolving +## @replaceable no +function scala_initialize +{ + if [[ ${SCALA_INITIALIZED} == true ]]; then + return + else + SCALA_INITIALIZED=true + fi + + if declare -f maven_add_install >/dev/null 2>&1; then + maven_add_install scaladoc + maven_add_install scalac + fi +} + +## @description initialize the scalac plug-in +## @audience private +## @stability evolving +## @replaceable no +function scalac_initialize +{ + scala_initialize +} + +## @description initialize the scaladoc plug-in +## @audience private +## @stability evolving +## @replaceable no +function scaladoc_initialize +{ + scala_initialize +} + +## @description +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function scalac_compile +{ + declare codebase=$1 + declare multijdkmode=$2 + + if ! verify_needed_test scalac; then + return 0 + fi + + if [[ ${codebase} = patch ]]; then + generic_postlog_compare compile scalac "${multijdkmode}" + fi +} + +## @description Count and compare the number of ScalaDoc warnings pre- and post- patch +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function scaladoc_rebuild +{ + declare codebase=$1 + + if [[ "${codebase}" = branch ]]; then + generic_pre_handler scaladoc false + else + generic_post_handler scaladoc scaladoc false true + fi +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function scalac_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "^/.*.scala:[0-9]*:" "${input}" > "${output}" +} + +## @description Helper for generic_logfilter +## @audience private +## @stability evolving +## @replaceable no +function scaladoc_logfilter +{ + declare input=$1 + declare output=$2 + + #shellcheck disable=SC2016,SC2046 + ${GREP} "^/.*.scala:[0-9]*:" "${input}" > "${output}" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/shellcheck.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/shellcheck.sh b/precommit/src/main/shell/test-patch.d/shellcheck.sh new file mode 100755 index 0000000..f0450f7 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/shellcheck.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no public APIs here +# SHELLDOC-IGNORE + +add_test_type shellcheck + +SHELLCHECK_TIMER=0 +SHELLCHECK_X=true + +SHELLCHECK=${SHELLCHECK:-$(which shellcheck 2>/dev/null)} + +# files that are going to get shellcheck'd +SHELLCHECK_CHECKFILES=() + +# files that are going to get shellcheck'd +SHELLCHECK_FILTERFILES=() + + +function shellcheck_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.sh$ ]]; then + # if it ends in an explicit .sh, then this is shell code. + add_test shellcheck + yetus_add_array_element SHELLCHECK_FILTERFILES "${filename}" + elif [[ ${BUILDTOOL} = maven && ${filename} =~ src/main/shell ]]; then + # if it is maven and in src/main/shell, assume it's shell code + add_test shellcheck + yetus_add_array_element SHELLCHECK_FILTERFILES "${filename}" + elif [[ ! ${filename} =~ \. ]]; then + # if it doesn't have an extension, then assume it is and + # we'll deal with it later + add_test shellcheck + yetus_add_array_element SHELLCHECK_FILTERFILES "${filename}" + fi +} + +function shellcheck_precheck +{ + declare langs + + if ! verify_command "shellcheck" "${SHELLCHECK}"; then + add_vote_table 0 shellcheck "Shellcheck was not available." + delete_test shellcheck + else + # shellcheck disable=SC2016 + SHELLCHECK_VERSION=$("${SHELLCHECK}" --version | "${GREP}" version: | "${AWK}" '{print $NF}') + + # versions less than 0.4.1 do not support -x + if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-9] || ${SHELLCHECK_VERSION} = 0.4.0 ]]; then + SHELLCHECK_X=false + fi + fi + + if [[ -z "${LANG}" ]]; then + langs=$(locale -a) + if [[ ${langs} =~ C.UTF-8 ]]; then + yetus_error "WARNING: shellcheck needs UTF-8 locale support. Forcing C.UTF-8." + export LANG=C.UTF-8 + export LC_ALL=C.UTF-8 + elif [[ ${langs} =~ en_US.UTF-8 ]]; then + yetus_error "WARNING: shellcheck needs UTF-8 locale support. Forcing en_US.UTF-8." + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + else + for i in ${langs}; do + if [[ "${i}" =~ UTF-8 ]]; then + yetus_error "WARNING: shellcheck needs UTF-8 locale support. Forcing ${i}." + export LANG="${i}" + export LC_ALL="${i}" + break + fi + done + fi + fi + + if [[ ! "${LANG}" =~ UTF-8 ]]; then + yetus_error "WARNING: shellcheck may fail without UTF-8 locale setting." + fi +} + +function shellcheck_criteria +{ + declare fn=$1 + declare text + + if [[ ! -f "${fn}" ]]; then + yetus_debug "Shellcheck rejected (not exist): ${fn}" + return + fi + + text=$(head -n 1 "${fn}") + + # shell check requires either a bangpath or a shell check directive + # on the first line. so check for a leading comment char + # and some sort of reference to 'sh' + if echo "${text}" | "${GREP}" -E -q "^#"; then + if echo "${text}" | "${GREP}" -q sh; then + yetus_add_array_element SHELLCHECK_CHECKFILES "${fn}" + yetus_debug "Shellcheck added: ${fn}" + fi + fi +} + +function shellcheck_findscripts +{ + declare fn + + # reset + SHELLCHECK_CHECKFILES=() + + # run through the files our filter caught + # this will set SHELLCHECK_CHECKFILES elements + for fn in "${SHELLCHECK_FILTERFILES[@]}"; do + shellcheck_criteria "${fn}" + done + + # finally, sort the array + yetus_sort_array SHELLCHECK_CHECKFILES +} + +function shellcheck_logic +{ + declare repostatus=$1 + declare i + + echo "Running shellcheck against all suspected shell scripts" + pushd "${BASEDIR}" >/dev/null + + # need to run this every time in case patch + # add/removed files + shellcheck_findscripts + + for i in "${SHELLCHECK_CHECKFILES[@]}"; do + if [[ "${SHELLCHECK_X}" = true ]]; then + "${SHELLCHECK}" -x -f gcc "${i}" >> "${PATCH_DIR}/${repostatus}-shellcheck-result.txt" + else + "${SHELLCHECK}" -f gcc "${i}" >> "${PATCH_DIR}/${repostatus}-shellcheck-result.txt" + fi + done + popd > /dev/null +} + +function shellcheck_preapply +{ + declare msg + + if ! verify_needed_test shellcheck; then + return 0 + fi + + big_console_header "shellcheck plugin: ${PATCH_BRANCH}" + + start_clock + + shellcheck_logic branch + + msg="v${SHELLCHECK_VERSION}" + if [[ ${SHELLCHECK_VERSION} =~ 0.[0-3].[0-5] ]]; then + msg="${msg} (This is an old version that has serious bugs. Consider upgrading.)" + fi + add_footer_table shellcheck "${msg}" + + # keep track of how much as elapsed for us already + SHELLCHECK_TIMER=$(stop_clock) + return 0 +} + +## @description Wrapper to call column_calcdiffs +## @audience private +## @stability evolving +## @replaceable no +## @param branchlog +## @param patchlog +## @return differences +function shellcheck_calcdiffs +{ + column_calcdiffs "$@" +} + +function shellcheck_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test shellcheck; then + return 0 + fi + + big_console_header "shellcheck plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${SHELLCHECK_TIMER}" + + shellcheck_logic patch + + calcdiffs \ + "${PATCH_DIR}/branch-shellcheck-result.txt" \ + "${PATCH_DIR}/patch-shellcheck-result.txt" \ + shellcheck \ + > "${PATCH_DIR}/diff-patch-shellcheck.txt" + + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/branch-shellcheck-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patch-shellcheck-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-shellcheck.txt" | ${AWK} '{print $1}') + + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_vote_table -1 shellcheck "${BUILDMODEMSG} ${statstring}" + add_footer_table shellcheck "@@BASE@@/diff-patch-shellcheck.txt" + bugsystem_linecomments "shellcheck" "${PATCH_DIR}/diff-patch-shellcheck.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 shellcheck "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 shellcheck "There were no new shellcheck issues." + return 0 +} + +function shellcheck_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + shellcheck_preapply + else + shellcheck_postapply + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/shelldocs.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/shelldocs.sh b/precommit/src/main/shell/test-patch.d/shelldocs.sh new file mode 100755 index 0000000..d52fa6b --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/shelldocs.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type shelldocs + +SHELLDOCS_TIMER=0 + +SHELLDOCS=${SHELLDOCS} +if [[ -z ${SHELLDOCS} ]]; then + for shelldocsexec in "${BINDIR}/shelldocs" "${BINDIR}/../shelldocs/shelldocs.py"; do + if [[ -f ${shelldocsexec} && -x ${shelldocsexec} ]]; then + SHELLDOCS=${shelldocsexec} + break + fi + done +fi + +SHELLDOCS_SPECIFICFILES="" + +function shelldocs_usage +{ + yetus_add_option "--shelldocs=<path>" "path to shelldocs executable" +} + +function shelldocs_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --shelldocs=*) + SHELLDOCS=${i#*=} + ;; + esac + done +} + +# if it ends in an explicit .sh, then this is shell code. +# if it doesn't have an extension, we assume it is shell code too +function shelldocs_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.sh$ ]]; then + add_test shelldocs + SHELLDOCS_SPECIFICFILES="${SHELLDOCS_SPECIFICFILES} ./${filename}" + fi + + if [[ ! ${filename} =~ \. ]]; then + add_test shelldocs + fi +} + +function shelldocs_precheck +{ + if ! verify_command "shelldocs" "${SHELLDOCS}"; then + add_vote_table 0 shelldocs "Shelldocs was not available." + delete_test shelldocs + fi +} + +function shelldocs_private_findbash +{ + declare i + declare value + declare list + + while read -r line; do + value=$(find "${line}" ! -name '*.cmd' -type f \ + | ${GREP} -E -v '(.orig$|.rej$)') + + for i in ${value}; do + if [[ ! ${i} =~ \.sh(\.|$) + && ! $(head -n 1 "${i}") =~ ^#! ]]; then + yetus_debug "Shelldocs skipped: ${i}" + continue + fi + list="${list} ${i}" + done + done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name scripts -o -type d -name libexec -o -type d -name shellprofile.d) + # shellcheck disable=SC2086 + echo ${list} ${SHELLDOCS_SPECIFICFILES} | tr ' ' '\n' | sort -u +} + +function shelldocs_preapply +{ + declare i + + if ! verify_needed_test shelldocs; then + return 0 + fi + + big_console_header "shelldocs plugin: ${PATCH_BRANCH}" + + start_clock + + echo "Running shelldocs against all identifiable shell scripts" + pushd "${BASEDIR}" >/dev/null + for i in $(shelldocs_private_findbash); do + if [[ -f ${i} ]]; then + ${SHELLDOCS} --input "${i}" --lint >> "${PATCH_DIR}/branch-shelldocs-result.txt" + fi + done + popd > /dev/null + + # keep track of how much as elapsed for us already + SHELLDOCS_TIMER=$(stop_clock) + return 0 +} + +function shelldocs_postapply +{ + declare i + declare numPrepatch + declare numPostpatch + declare diffPostpatch + declare fixedpatch + declare statstring + + if ! verify_needed_test shelldocs; then + return 0 + fi + + big_console_header "shelldocs plugin: ${BUILDMODE}" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${SHELLDOCS_TIMER}" + + echo "Running shelldocs against all identifiable shell scripts" + # we re-check this in case one has been added + for i in $(shelldocs_private_findbash); do + if [[ -f ${i} ]]; then + ${SHELLDOCS} --input "${i}" --lint >> "${PATCH_DIR}/patch-shelldocs-result.txt" + fi + done + + calcdiffs \ + "${PATCH_DIR}/branch-shelldocs-result.txt" \ + "${PATCH_DIR}/patch-shelldocs-result.txt" \ + shelldocs \ + > "${PATCH_DIR}/diff-patch-shelldocs.txt" + + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/branch-shelldocs-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patch-shelldocs-result.txt" | ${AWK} '{print $1}') + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-shelldocs.txt" | ${AWK} '{print $1}') + + ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch)) + + statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" ) + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_vote_table -1 shelldocs "${BUILDMODEMSG} ${statstring}" + add_footer_table shelldocs "@@BASE@@/diff-patch-shelldocs.txt" + bugsystem_linecomments "shelldocs" "${PATCH_DIR}/diff-patch-shelldocs.txt" + return 1 + elif [[ ${fixedpatch} -gt 0 ]]; then + add_vote_table +1 shelldocs "${BUILDMODEMSG} ${statstring}" + return 0 + fi + + add_vote_table +1 shelldocs "There were no new shelldocs issues." + return 0 +} + +function shelldocs_postcompile +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + shelldocs_preapply + else + shelldocs_postapply + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/tap.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/tap.sh b/precommit/src/main/shell/test-patch.d/tap.sh new file mode 100755 index 0000000..465b7f1 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/tap.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_format tap + +TAP_FAILED_TESTS="" +TAP_LOG_DIR="target/tap" + +function tap_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --tap-log-dir=*) + TAP_LOG_DIR=${i#=*} + ;; + esac + done +} + +function tap_usage +{ + yetus_add_option "--tap-log-dir=<dir>" "Directory relative to the module for tap output (default: \"target/tap\")" +} + +function tap_process_tests +{ + # shellcheck disable=SC2034 + declare module=$1 + # shellcheck disable=SC2034 + declare buildlogfile=$2 + declare filefrag=$3 + declare result=0 + declare module_failed_tests + declare filenames + + if [[ -d "${TAP_LOG_DIR}" ]]; then + filenames=$(find "${TAP_LOG_DIR}" -type f -exec "${GREP}" -l -E "^not ok" {} \;) + fi + + if [[ -n "${filenames}" ]]; then + module_failed_tests=$(echo "${filenames}" \ + | ${SED} -e "s,${TAP_LOG_DIR},,g" -e s,^/,,g ) + # shellcheck disable=SC2086 + cat ${filenames} >> "${PATCH_DIR}/patch-${filefrag}.tap" + TAP_LOGS="${TAP_LOGS} @@BASE@@/patch-${filefrag}.tap" + TAP_FAILED_TESTS="${TAP_FAILED_TESTS} ${module_failed_tests}" + ((result=result+1)) + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +function tap_finalize_results +{ + declare jdk=$1 + + if [[ -n "${TAP_FAILED_TESTS}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "${jdk}Failed TAP tests" ${TAP_FAILED_TESTS} + TAP_FAILED_TESTS="" + add_footer_table "TAP logs" "${TAP_LOGS}" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/test4tests.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/test4tests.sh b/precommit/src/main/shell/test-patch.d/test4tests.sh new file mode 100755 index 0000000..6c3230c --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/test4tests.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type test4tests + +## @description Check the patch file for changed/new tests +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function test4tests_patchfile +{ + declare testReferences=0 + declare i + + if [[ "${BUILDMODE}" = full ]]; then + return + fi + + big_console_header "Checking there are new or changed tests in the patch." + + if ! verify_needed_test unit; then + echo "Patch does not appear to need new or modified tests." + return 0 + fi + + start_clock + + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ (^|/)test/ ]]; then + ((testReferences=testReferences + 1)) + fi + done + + echo "There appear to be ${testReferences} test file(s) referenced in the patch." + if [[ ${testReferences} == 0 ]] ; then + add_vote_table -1 "test4tests" \ + "The patch doesn't appear to include any new or modified tests. " \ + "Please justify why no new tests are needed for this patch." \ + "Also please list what manual steps were performed to verify this patch." + return 1 + fi + add_vote_table +1 "test4tests" \ + "The patch appears to include ${testReferences} new or modified test files." + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/unitveto.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/unitveto.sh b/precommit/src/main/shell/test-patch.d/unitveto.sh new file mode 100755 index 0000000..e0d348e --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/unitveto.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_type unitveto + +UNITVETO_RE=${UNITVETO_RE:-} + +function unitveto_filefilter +{ + declare filename=$1 + + if [[ -n "${UNITVETO_RE}" + && ${filename} =~ ${UNITVETO_RE} ]]; then + yetus_debug "unitveto: ${filename} matched" + add_test unitveto + fi +} + +function unitveto_usage +{ + yetus_add_option "--unitveto-re=<regex>" "Regex to automatically -1 due to manual test requirements" +} + +function unitveto_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --unitveto-re=*) + UNITVETO_RE=${i#*=} + ;; + esac + done +} + +function unitveto_patchfile +{ + if ! verify_needed_test unit; then + return 0 + fi + + if ! verify_needed_test unitveto; then + return 0 + fi + + add_vote_table -1 unitveto "Patch requires manual testing." + return 1 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/whitespace.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/whitespace.sh b/precommit/src/main/shell/test-patch.d/whitespace.sh new file mode 100755 index 0000000..390f15f --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/whitespace.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +WHITESPACE_EOL_IGNORE_LIST= +WHITESPACE_TABS_IGNORE_LIST=Makefile + +add_test_type whitespace + +## @description whitespace usage hook +## @audience private +## @stability evolving +## @replaceable no +function whitespace_usage +{ + yetus_add_option "--whitespace-eol-ignore-list=<list>" "comma-separated regex list of filenames to ignore on checking whitespaces at EOL (default '${WHITESPACE_EOL_IGNORE_LIST}')" + yetus_add_option "--whitespace-tabs-ignore-list=<list>" "comma-separated regex list of filenames to ignore on checking tabs in a file (default '${WHITESPACE_TABS_IGNORE_LIST}')" +} + +## @description whitespace parse args hook +## @audience private +## @stability evolving +## @replaceable no +function whitespace_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --whitespace-eol-ignore-list=*) + yetus_comma_to_array WHITESPACE_EOL_IGNORE_LIST "${i#*=}" + ;; + --whitespace-tabs-ignore-list=*) + yetus_comma_to_array WHITESPACE_TABS_IGNORE_LIST "${i#*=}" + ;; + esac + done +} + +function whitespace_linecomment_reporter +{ + declare file=$1 + shift + declare comment=$* + declare tmpfile="${PATCH_DIR}/wlr.$$.${RANDOM}" + + while read -r line; do + { + # shellcheck disable=SC2086 + printf "%s" "$(echo ${line} | cut -f1-2 -d:)" + echo "${comment}" + } >> "${tmpfile}" + done < "${file}" + + bugsystem_linecomments "whitespace:" "${tmpfile}" + rm "${tmpfile}" +} + +function whitespace_postcompile +{ + declare repostatus=$1 + declare count + declare result=0 + declare eolignore + declare tabsignore + + if [[ "${repostatus}" = branch ]]; then + return 0 + fi + + big_console_header "Checking for whitespace issues." + start_clock + + pushd "${BASEDIR}" >/dev/null + + eolignore=$(printf -- "-e ^%s: " "${WHITESPACE_EOL_IGNORE_LIST[@]}") + tabsignore=$(printf -- "-e ^%s: " "${WHITESPACE_TABS_IGNORE_LIST[@]}") + + case "${BUILDMODE}" in + patch) + # shellcheck disable=SC2016,SC2086 + ${AWK} '/\t/ {print $0}' \ + "${GITDIFFCONTENT}" \ + | ${GREP} -v ${tabsignore} >> "${PATCH_DIR}/whitespace-tabs.txt" + + # shellcheck disable=SC2086 + ${GREP} -E '[[:blank:]]$' \ + "${GITDIFFCONTENT}" \ + | ${GREP} -v ${eolignore} >> "${PATCH_DIR}/whitespace-eol.txt" + ;; + full) + # shellcheck disable=SC2086 + ${GIT} grep -n -I --extended-regexp '[[:blank:]]$' \ + | "${GREP}" -v ${eolignore} \ + >> "${PATCH_DIR}/whitespace-eol.txt" + # shellcheck disable=SC2086 + ${GIT} grep -n -I $'\t' \ + | "${GREP}" -v ${tabsignore} \ + >> "${PATCH_DIR}/whitespace-tabs.txt" + ;; + esac + + # shellcheck disable=SC2016 + count=$(wc -l "${PATCH_DIR}/whitespace-eol.txt" | ${AWK} '{print $1}') + + if [[ ${count} -gt 0 ]]; then + if [[ "${BUILDMODE}" = full ]]; then + add_vote_table -1 whitespace "${BUILDMODEMSG} has ${count} line(s) that end in whitespace." + else + add_vote_table -1 whitespace \ + "${BUILDMODEMSG} has ${count} line(s) that end in whitespace. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply"; + fi + + whitespace_linecomment_reporter "${PATCH_DIR}/whitespace-eol.txt" "end of line" + add_footer_table whitespace "@@BASE@@/whitespace-eol.txt" + ((result=result+1)) + fi + + # shellcheck disable=SC2016 + count=$(wc -l "${PATCH_DIR}/whitespace-tabs.txt" | ${AWK} '{print $1}') + + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 whitespace "${BUILDMODEMSG} ${count}"\ + " line(s) with tabs." + add_footer_table whitespace "@@BASE@@/whitespace-tabs.txt" + whitespace_linecomment_reporter "${PATCH_DIR}/whitespace-tabs.txt" "tabs in line" + ((result=result+1)) + fi + + if [[ ${result} -gt 0 ]]; then + popd >/dev/null + return 1 + fi + + popd >/dev/null + add_vote_table +1 whitespace "${BUILDMODEMSG} has no whitespace issues." + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/test-patch.d/xml.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/test-patch.d/xml.sh b/precommit/src/main/shell/test-patch.d/xml.sh new file mode 100755 index 0000000..1e56905 --- /dev/null +++ b/precommit/src/main/shell/test-patch.d/xml.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare -a XML_FILES + +add_test_type xml + +function xml_filefilter +{ + declare filename=$1 + + if [[ ${filename} =~ \.xml$ ]]; then + add_test xml + fi +} + +function xml_precheck +{ + if ! verify_command "jrunscript" "${JAVA_HOME}/bin/jrunscript"; then + add_vote_table 0 xml "jrunscript was not available." + delete_test xml + fi +} + +function xml_postcompile +{ + declare repostatus=$1 + declare js + declare i + declare count + + if ! verify_needed_test xml; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + return 0 + fi + + big_console_header "XML verification: ${BUILDMODE}" + + js="${JAVA_HOME}/bin/jrunscript" + + start_clock + + pushd "${BASEDIR}" >/dev/null + for i in "${CHANGED_FILES[@]}"; do + if [[ ${i} =~ \.xml$ && -f ${i} ]]; then + if ! "${js}" -e "XMLDocument(arguments[0])" "${i}" > "${PATCH_DIR}/xml.txt.tmp" 2>&1; then + { + echo "" + echo "${i}:" + echo "" + cat "${PATCH_DIR}/xml.txt.tmp" + } >> "${PATCH_DIR}/xml.txt" + ((count=count+1)) + XML_FILES+=("${i}") + fi + fi + done + + popd >/dev/null + + if [[ -f "${PATCH_DIR}/xml.txt.tmp" ]]; then + rm "${PATCH_DIR}/xml.txt.tmp" + fi + + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 xml "${BUILDMODEMSG} has ${count} ill-formed XML file(s)." + add_footer_table xml "@@BASE@@/xml.txt" + populate_test_table "XML" "Parsing Error(s):" "${XML_FILES[@]}" + return 1 + fi + + add_vote_table +1 xml "${BUILDMODEMSG} has no ill-formed XML file." + return 0 +}
