http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/core.d/builtin-bugsystem.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/builtin-bugsystem.sh b/precommit/core.d/builtin-bugsystem.sh new file mode 100755 index 0000000..f6e25f1 --- /dev/null +++ b/precommit/core.d/builtin-bugsystem.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This bug system handles the output on the screen. + +add_bugsystem console + +## @description Print out the finished details on the console +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +## @return 0 on success +## @return 1 on failure +function console_finalreport +{ + declare result=$1 + shift + declare i=0 + declare ourstring + declare vote + declare subs + declare ela + declare comment + declare commentfile1="${PATCH_DIR}/comment.1" + declare commentfile2="${PATCH_DIR}/comment.2" + declare normaltop + declare line + declare seccoladj=0 + declare spcfx=${PATCH_DIR}/spcl.txt + + if [[ ${result} == 0 ]]; then + if [[ ${JENKINS} == false ]]; then + if declare -f ${PROJECT_NAME}_console_success >/dev/null; then + ${PROJECT_NAME}_console_success > "${spcfx}" + else + { + printf "IF9fX18gICAgICAgICAgICAgICAgICAgICAgICAgICAgICBfIAovIF9fX3wg"; + printf "XyAgIF8gIF9fXyBfX18gX19fICBfX18gX19ffCB8ClxfX18gXHwgfCB8IHwv"; + printf "IF9fLyBfXy8gXyBcLyBfXy8gX198IHwKIF9fXykgfCB8X3wgfCAoX3wgKF98"; + printf "ICBfXy9cX18gXF9fIFxffAp8X19fXy8gXF9fLF98XF9fX1xfX19cX19ffHxf"; + printf "X18vX19fKF8pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg"; + printf "ICAK"; + } > "${spcfx}" + fi + fi + printf "\n\n+1 overall\n\n" + else + if [[ ${JENKINS} == false ]]; then + if declare -f ${PROJECT_NAME}_console_failure >/dev/null; then + ${PROJECT_NAME}_console_failure > "${spcfx}" + else + { + printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf"; + printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf"; + printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98"; + printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg"; + printf "ICAgICAgICAgICAgICAgICAK" + } > "${spcfx}" + fi + fi + printf "\n\n-1 overall\n\n" + fi + + if [[ -f ${spcfx} ]]; then + if which base64 >/dev/null 2>&1; then + base64 --decode "${spcfx}" 2>/dev/null + elif which openssl >/dev/null 2>&1; then + openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null + fi + echo + echo + rm "${spcfx}" + fi + + seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}") + if [[ ${seccoladj} -lt 10 ]]; then + seccoladj=10 + fi + + seccoladj=$((seccoladj + 2 )) + i=0 + until [[ $i -eq ${#TP_HEADER[@]} ]]; do + printf "%s\n" "${TP_HEADER[${i}]}" + ((i=i+1)) + done + + printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment" + echo "============================================================================" + i=0 + until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + comment=$(echo "${ourstring}" | cut -f5 -d\|) + + echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}" + normaltop=$(head -1 "${commentfile1}") + ${SED} -e '1d' "${commentfile1}" > "${commentfile2}" + + printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \ + "${subs}" "${ela}" "${normaltop}" + while read -r line; do + printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}" + done < "${commentfile2}" + + ((i=i+1)) + rm "${commentfile2}" "${commentfile1}" 2>/dev/null + done + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}") + printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason" + i=0 + until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do + ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}" + ((i=i+1)) + done + fi + + printf "\n\n|| Subsystem || Report/Notes ||\n" + echo "============================================================================" + i=0 + + until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${TP_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") + printf "%s\n" "${comment}" + ((i=i+1)) + done +}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/core.d/builtin-personality.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/builtin-personality.sh b/precommit/core.d/builtin-personality.sh new file mode 100755 index 0000000..1258d46 --- /dev/null +++ b/precommit/core.d/builtin-personality.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function personality_modules +{ + if declare -f "${BUILDTOOL}_builtin_personality_modules" >/dev/null; then + "${BUILDTOOL}_builtin_personality_modules" "$@" + fi +} + +function personality_file_tests +{ + if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then + "${BUILDTOOL}_builtin_personality_file_tests" "$@" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/core.d/patchfiles.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/patchfiles.sh b/precommit/core.d/patchfiles.sh new file mode 100755 index 0000000..7bb2f8b --- /dev/null +++ b/precommit/core.d/patchfiles.sh @@ -0,0 +1,330 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description Use curl to download the patch as a last resort +## @audience private +## @stability evolving +## @param patchloc +## @param output +## @return 0 got something +## @return 1 error +function generic_locate_patch +{ + declare input=$1 + declare output=$2 + + if [[ "${OFFLINE}" == true ]]; then + yetus_debug "generic_locate_patch: offline, skipping" + return 1 + fi + + ${CURL} --silent \ + --output "${output}" \ + "${input}" + if [[ $? != 0 ]]; then + yetus_debug "generic_locate_patch: failed to download the patch." + return 1 + fi + return 0 +} + +## @description Given a possible patch file, guess if it's a patch file +## @description only using the more intense verify if we really need to +## @audience private +## @stability evolving +## @param path to patch file to test +## @return 0 we think it's a patch file +## @return 1 we think it's not a patch file +function guess_patch_file +{ + declare patch=$1 + declare fileOutput + + if [[ ! -f ${patch} ]]; then + return 1 + fi + + yetus_debug "Trying to guess if ${patch} is a patch file." + fileOutput=$("${FILE}" "${patch}") + if [[ $fileOutput =~ \ diff\ ]]; then + yetus_debug "file magic says it's a diff." + return 0 + fi + + fileOutput=$(head -n 1 "${patch}" | "${GREP}" -E "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$") + if [[ $? == 0 ]]; then + yetus_debug "first line looks like a patch file." + return 0 + fi + + patchfile_dryrun_driver "${patch}" +} + +## @description Given ${PATCH_OR_ISSUE}, determine what type of patch file is in use, +## @description and do the necessary work to place it into ${PATCH_DIR}/patch. +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure, may exit +function locate_patch +{ + declare bugsys + declare patchfile="" + declare gotit=false + + yetus_debug "locate patch" + + if [[ -z "${PATCH_OR_ISSUE}" ]]; then + yetus_error "ERROR: No patch provided." + cleanup_and_exit 1 + fi + + echo "Processing: ${PATCH_OR_ISSUE}" + # it's a declarely provided file + if [[ -f ${PATCH_OR_ISSUE} ]]; then + patchfile="${PATCH_OR_ISSUE}" + else + # run through the bug systems. maybe they know? + for bugsys in ${BUGSYSTEMS}; do + if declare -f ${bugsys}_locate_patch >/dev/null 2>&1; then + "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" + if [[ $? == 0 ]]; then + gotit=true + fi + fi + done + + # ok, none of the bug systems know. let's see how smart we are + if [[ ${gotit} == false ]]; then + generic_locate_patch "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" + if [[ $? != 0 ]]; then + yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}." + cleanup_and_exit 1 + fi + fi + fi + + if [[ ! -f "${PATCH_DIR}/patch" + && -f "${patchfile}" ]]; then + cp "${patchfile}" "${PATCH_DIR}/patch" + if [[ $? == 0 ]] ; then + echo "Patch file ${patchfile} copied to ${PATCH_DIR}" + else + yetus_error "ERROR: Could not copy ${patchfile} to ${PATCH_DIR}" + cleanup_and_exit 1 + fi + fi +} + +## @description if patch-level zero, then verify we aren't +## @description just adding files +## @audience public +## @stability stable +## @param log filename +## @replaceable no +## @returns $? +function patchfile_verify_zero +{ + declare logfile=$1 + shift + declare dir + declare changed_files1 + declare changed_files2 + declare filename + + # don't return /dev/null + # shellcheck disable=SC2016 + changed_files1=$(${AWK} 'function p(s){if(s!~"^/dev/null"){print s}} + /^diff --git / { p($3); p($4) } + /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u) + + # maybe we interpreted the patch wrong? check the log file + # shellcheck disable=SC2016 + changed_files2=$(${GREP} -E '^[cC]heck' "${logfile}" \ + | ${AWK} '{print $3}' \ + | ${SED} -e 's,\.\.\.$,,g') + + for filename in ${changed_files1} ${changed_files2}; do + + # leading prefix = bad + if [[ ${filename} =~ ^(a|b)/ ]]; then + return 1 + fi + + # touching an existing file is proof enough + # that pl=0 is good + if [[ -f ${filename} ]]; then + return 0 + fi + + dir=$(dirname "${filename}" 2>/dev/null) + if [[ -n ${dir} && -d ${dir} ]]; then + return 0 + fi + done + + # ¯\_(ã)_/¯ - no way for us to know, all new files with no prefix! + yetus_error "WARNING: Patch only adds files; using patch level ${PATCH_LEVEL}" + return 0 +} + +## @description git apply dryrun +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to dryrun +function gitapply_dryrun +{ + declare patchfile=$1 + declare prefixsize=${2:-0} + + while [[ ${prefixsize} -lt 4 + && -z ${PATCH_METHOD} ]]; do + yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \ + "${GIT}" apply --binary -v --check "-p${prefixsize}" "${patchfile}" + if [[ $? == 0 ]]; then + PATCH_LEVEL=${prefixsize} + PATCH_METHOD=gitapply + break + fi + ((prefixsize=prefixsize+1)) + done + + if [[ ${prefixsize} -eq 0 ]]; then + patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log" + if [[ $? != 0 ]]; then + PATCH_METHOD="" + PATCH_LEVEL="" + gitapply_dryrun "${patchfile}" 1 + fi + fi +} + +## @description patch patch dryrun +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to dryrun +function patchcmd_dryrun +{ + declare patchfile=$1 + declare prefixsize=${2:-0} + + while [[ ${prefixsize} -lt 4 + && -z ${PATCH_METHOD} ]]; do + # shellcheck disable=SC2153 + yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \ + "${PATCH}" "-p${prefixsize}" -E --dry-run < "${patchfile}" + if [[ $? == 0 ]]; then + PATCH_LEVEL=${prefixsize} + PATCH_METHOD=patchcmd + break + fi + ((prefixsize=prefixsize+1)) + done + + if [[ ${prefixsize} -eq 0 ]]; then + patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log" + if [[ $? != 0 ]]; then + PATCH_METHOD="" + PATCH_LEVEL="" + patchcmd_dryrun "${patchfile}" 1 + fi + fi +} + +## @description driver for dryrun methods +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to dryrun +function patchfile_dryrun_driver +{ + declare patchfile=$1 + declare method + + #shellcheck disable=SC2153 + for method in "${PATCH_METHODS[@]}"; do + if declare -f ${method}_dryrun >/dev/null; then + "${method}_dryrun" "${patchfile}" + fi + if [[ -n ${PATCH_METHOD} ]]; then + break + fi + done + + if [[ -n ${PATCH_METHOD} ]]; then + return 0 + fi + return 1 +} + +## @description git patch apply +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to apply +function gitapply_apply +{ + declare patchfile=$1 + declare extraopts + + if [[ "${COMMITMODE}" = true ]]; then + extraopts="--whitespace=fix" + fi + + echo "Applying the patch:" + yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-apply.log" \ + "${GIT}" apply --binary ${extraopts} -v --stat --apply "-p${PATCH_LEVEL}" "${patchfile}" + ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-apply.log" +} + +## @description patch patch apply +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to apply +function patchcmd_apply +{ + declare patchfile=$1 + + echo "Applying the patch:" + yetus_run_and_redirect "${PATCH_DIR}/apply-patch-patch-apply.log" \ + "${PATCH}" "-p${PATCH_LEVEL}" -E < "${patchfile}" + cat "${PATCH_DIR}/apply-patch-patch-apply.log" +} + +## @description driver for patch apply methods +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to apply +function patchfile_apply_driver +{ + declare patchfile=$1 + + if declare -f ${PATCH_METHOD}_apply >/dev/null; then + "${PATCH_METHOD}_apply" "${patchfile}" + if [[ $? -gt 0 ]]; then + return 1 + fi + else + yetus_error "ERROR: Patching method ${PATCH_METHOD} does not have a way to apply patches!" + return 1 + fi + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/bigtop.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/bigtop.sh b/precommit/personality/bigtop.sh new file mode 100755 index 0000000..3854f6b --- /dev/null +++ b/precommit/personality/bigtop.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-checkstyle,-findbugs" + +function personality_globals +{ + # shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + # shellcheck disable=SC2034 + JIRA_ISSUE_RE='^BIGTOP-[0-9]+$' + # shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="" + # shellcheck disable=SC2034 + BUILDTOOL=gradle + # shellcheck disable=SC2034 + GITHUB_REPO="apache/bigtop" + # shellcheck disable=SC2034 + BIGTOP_PUPPETSETUP=false +} + +add_test_type bigtop + +function bigtop_usage +{ + echo "Bigtop specific:" + echo "--bigtop-puppet=[false|true] execute the bigtop puppet setup (needs sudo to root)" +} + +function bigtop_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --bigtop-puppet=*) + BIGTOP_PUPPETSETUP=${i#*=} + ;; + esac + done +} + +function bigtop_precheck_postinstall +{ + if [[ ${BIGTOP_PUPPETSETUP} = "true" ]]; then + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/bigtop-branch-toolchain.txt" "${GRADLEW}" toolchain + popd >/dev/null + fi +} + +function bigtop_postapply_postinstall +{ + if [[ ${BIGTOP_PUPPETSETUP} = "true" ]]; then + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/bigtop-patch-toolchain.txt" "${GRADLEW}" toolchain + popd >/dev/null + fi +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/flink.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/flink.sh b/precommit/personality/flink.sh new file mode 100755 index 0000000..356c76b --- /dev/null +++ b/precommit/personality/flink.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^FLINK-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/flink" + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="" +} + +add_test_type flinklib + +function flinklib_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ + || ${filename} =~ \.scala$ + || ${filename} =~ pom.xml$ ]]; then + add_test flinklib + fi +} + +function flinklib_count +{ + find "${BASEDIR}" \ + | ${GREP} "/lib/" \ + | ${GREP} -v "_qa_workdir" \ + | wc -l +} + +function flinklib_preapply +{ + start_clock + big_console_header "${PATCH_BRANCH} flink library dependencies" + + verify_needed_test flinklib + if [[ $? == 0 ]]; then + echo "Patch does not need flinklib testing." + return 0 + fi + + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/branch-flinklib-root.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch + if [[ $? != 0 ]]; then + add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}." + fi + FLINK_PRE_LIB_FILES=$(flinklib_count) + popd >/dev/null +} + +function flinklib_postapply +{ + start_clock + big_console_header "Patch flink library dependencies" + + verify_needed_test flinklib + if [[ $? == 0 ]]; then + echo "Patch does not need flinklib testing." + return 0 + fi + + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/patch-flinklib-root.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch + FLINK_POST_LIB_FILES=$(flinklib_count) + popd >/dev/null + + + if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then + add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \ + "${FLINK_PRE_LIB_FILES} to ${FLINK_POST_LIB_FILES}" + return 1 + elif [[ "${FLINK_POST_LIB_FILES}" -eq "${FLINK_PRE_LIB_FILES}" ]]; then + add_vote_table 0 flinklib "Patch did not change lib dependencies" \ + " (still ${FLINK_PRE_LIB_FILES})" + else + add_vote_table +1 flinklib "Patch decreases lib folder dependencies by " \ + "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))." + fi + return 0 +} + +function flinklib_rebuild +{ + declare repostatus=$1 + + if [[ "${repostatus}" = branch ]]; then + flinklib_preapply + else + flinklib_postinstall + fi +} + http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/geode.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/geode.sh b/precommit/personality/geode.sh new file mode 100644 index 0000000..f16e431 --- /dev/null +++ b/precommit/personality/geode.sh @@ -0,0 +1,18 @@ +personality_plugins "all,-ant,-javac,-scalac,-scaladoc" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=develop + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/GEODE/How+to+Contribute"; + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^(GEODE)-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/incubator-geode" + #shellcheck disable=SC2034 + BUILDTOOL=gradle +# PYLINT_OPTIONS="--indent-string=' '" + +# HADOOP_MODULES="" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/hadoop.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh new file mode 100755 index 0000000..f958f0c --- /dev/null +++ b/precommit/personality/hadoop.sh @@ -0,0 +1,376 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Override these to match Apache Hadoop's requirements + +personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"; + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/hadoop" + #shellcheck disable=SC2034 + PYLINT_OPTIONS="--indent-string=' '" + + HADOOP_MODULES="" +} + +function hadoop_module_manipulation +{ + local startingmodules=${1:-normal} + local module + local hdfs_modules + local mapred_modules + local yarn_modules + local ordered_modules + local tools_modules + local passed_modules + local flags + + yetus_debug "hmm in: ${startingmodules}" + + if [[ ${startingmodules} = normal ]]; then + startingmodules=${CHANGED_MODULES} + elif [[ ${startingmodules} = union ]]; then + startingmodules=${CHANGED_UNION_MODULES} + fi + + yetus_debug "hmm expanded to: ${startingmodules}" + + if [[ ${startingmodules} = "." ]]; then + yetus_debug "hmm shortcut since ." + HADOOP_MODULES=. + return + fi + + # ${startingmodules} is already sorted and uniq'd. + # let's remove child modules if we're going to + # touch their parent. + passed_modules=${startingmodules} + for module in ${startingmodules}; do + yetus_debug "Stripping ${module}" + # shellcheck disable=SC2086 + passed_modules=$(echo ${passed_modules} | tr ' ' '\n' | ${GREP} -v ${module}/ ) + done + + yetus_debug "hmm pre-ordering: ${startingmodules}" + + # Hadoop's compilation rules are complex + # Precedence: common > [hdfs|yarn] > mapred > tools + # - everything depends upon common + # - hdfs needs common's native bits for unit tests + # - mapred depends upon yarn since it is a yarn app + # - tools can require anything + + for module in ${passed_modules}; do + yetus_debug "Personality ordering ${module}" + if [[ ${module} = "." ]]; then + HADOOP_MODULES=. + break + fi + + if [[ ${module} = hadoop-hdfs-project* ]]; then + hdfs_modules="${hdfs_modules} ${module}" + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then + ordered_modules="${ordered_modules} ${module}" + elif [[ ${module} = hadoop-tools* ]]; then + tools_modules="${tools_modules} ${module}" + elif [[ ${module} = hadoop-mapreduce-project* ]]; then + mapred_modules="${mapred_modules} ${module}" + elif [[ ${module} = hadoop-yarn-project* ]]; then + yarn_modules="${yarn_modules} ${module}" + else + ordered_modules="${ordered_modules} ${module}" + fi + done + + HADOOP_MODULES="${ordered_modules} ${hdfs_modules} ${yarn_modules} ${mapred_modules} ${tools_modules}" + + yetus_debug "hmm out: ${HADOOP_MODULES}" +} + +function hadoop_unittest_prereqs +{ + local need_common=0 + local building_common=0 + local module + local flags + local fn + + for module in ${HADOOP_MODULES}; do + if [[ ${module} = hadoop-hdfs-project* ]]; then + need_common=1 + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then + building_common=1 + fi + done + + if [[ ${need_common} -eq 1 + && ${building_common} -eq 0 ]]; then + echo "unit test pre-reqs:" + module="hadoop-common-project/hadoop-common" + fn=$(module_file_fragment "${module}") + flags=$(hadoop_native_flags) + pushd "${BASEDIR}/${module}" >/dev/null + # shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} + popd >/dev/null + fi +} + +function hadoop_native_flags +{ + + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + # Based upon HADOOP-11937 + # + # Some notes: + # + # - getting fuse to compile on anything but Linux + # is always tricky. + # - Darwin assumes homebrew is in use. + # - HADOOP-12027 required for bzip2 on OS X. + # - bzip2 is broken in lots of places. + # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 + # + + # current build servers are pretty limited in + # what they support + if [[ ${JENKINS} = true + && ${DOCKERSUPPORT} = false ]]; then + # shellcheck disable=SC2086 + echo -Pnative \ + -Drequire.snappy -Drequire.openssl -Drequire.fuse \ + -Drequire.test.libhadoop + return + fi + + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo -Pnative -Drequire.libwebhdfs \ + -Drequire.snappy -Drequire.openssl -Drequire.fuse \ + -Drequire.test.libhadoop + ;; + Darwin) + JANSSON_INCLUDE_DIR=/usr/local/opt/jansson/include + JANSSON_LIBRARY=/usr/local/opt/jansson/lib + export JANSSON_LIBRARY JANSSON_INCLUDE_DIR + # shellcheck disable=SC2086 + echo \ + -Pnative -Drequire.snappy \ + -Drequire.openssl \ + -Dopenssl.prefix=/usr/local/opt/openssl/ \ + -Dopenssl.include=/usr/local/opt/openssl/include \ + -Dopenssl.lib=/usr/local/opt/openssl/lib \ + -Drequire.libwebhdfs -Drequire.test.libhadoop + ;; + *) + # shellcheck disable=SC2086 + echo \ + -Pnative \ + -Drequire.snappy -Drequire.openssl \ + -Drequire.libwebhdfs -Drequire.test.libhadoop + ;; + esac +} + +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + local extra="" + local ordering="normal" + local needflags=false + local flags + local fn + local i + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + asflicense) + # this is very fast and provides the full path if we do it from + # the root of the source + personality_enqueue_module . + return + ;; + checkstyle) + ordering="union" + extra="-DskipTests" + ;; + compile) + ordering="union" + extra="-DskipTests" + needflags=true + + # if something in common changed, we build the whole world + if [[ ${CHANGED_MODULES} =~ hadoop-common ]]; then + yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " + ordering="." + fi + ;; + distclean) + ordering="." + extra="-DskipTests" + ;; + javadoc) + if [[ ${repostatus} = patch ]]; then + echo "javadoc pre-reqs:" + for i in hadoop-project \ + hadoop-common-project/hadoop-annotations; do + fn=$(module_file_fragment "${i}") + pushd "${BASEDIR}/${i}" >/dev/null + echo "cd ${i}" + echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install + popd >/dev/null + done + fi + extra="-Pdocs -DskipTests" + ;; + mvninstall) + extra="-DskipTests" + if [[ ${repostatus} = branch ]]; then + ordering=. + fi + ;; + unit) + if [[ ${TEST_PARALLEL} = "true" ]] ; then + extra="-Pparallel-tests" + if [[ -n ${TEST_THREADS:-} ]]; then + extra="${extra} -DtestsThreadCount=${TEST_THREADS}" + fi + fi + needflags=true + hadoop_unittest_prereqs + + verify_needed_test javac + if [[ $? == 0 ]]; then + yetus_debug "hadoop: javac not requested" + verify_needed_test native + if [[ $? == 0 ]]; then + yetus_debug "hadoop: native not requested" + yetus_debug "hadoop: adding -DskipTests to unit test" + extra="-DskipTests" + fi + fi + + verify_needed_test shellcheck + if [[ $? == 0 + && ! ${CHANGED_FILES} =~ \.bats ]]; then + yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile" + extra="${extra} -P!shelltest" + else + extra="${extra} -Pshelltest" + fi + ;; + *) + extra="-DskipTests" + ;; + esac + + if [[ ${needflags} = true ]]; then + flags=$(hadoop_native_flags) + extra="${extra} ${flags}" + fi + + hadoop_module_manipulation ${ordering} + + for module in ${HADOOP_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +function personality_file_tests +{ + local filename=$1 + + yetus_debug "Using Hadoop-specific personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/scripts + || ${filename} =~ src/test/scripts + ]]; then + yetus_debug "tests/shell: ${filename}" + add_test unit + elif [[ ${filename} =~ \.md$ + || ${filename} =~ \.md\.vm$ + || ${filename} =~ src/site + ]]; then + yetus_debug "tests/site: ${filename}" + add_test site + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test cc + add_test unit + add_test javac + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + || ${filename} =~ src/main + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test javac + add_test javadoc + add_test mvninstall + add_test unit + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +function hadoop_console_success +{ + printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; + printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; + printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; + printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; + printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; + printf "ICB8bXwgICB8bXwgIAo" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/hbase.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/hbase.sh b/precommit/personality/hbase.sh new file mode 100755 index 0000000..8451f17 --- /dev/null +++ b/precommit/personality/hbase.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^HBASE-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/hbase" + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="" + + # All supported Hadoop versions that we want to test the compilation with + HBASE_HADOOP_VERSIONS="2.4.1 2.5.2 2.6.0" + + # Override the maven options + MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}" +} + +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + local extra="" + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + extra="-DHBasePatchProcess" + + if [[ ${repostatus} == branch + && ${testtype} == mvninstall ]];then + personality_enqueue_module . ${extra} + return + fi + + if [[ ${testtype} = findbugs ]]; then + for module in ${CHANGED_MODULES}; do + # skip findbugs on hbase-shell + if [[ ${module} == hbase-shell ]]; then + continue + else + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + fi + done + return + fi + + for module in ${CHANGED_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +################################################### + +add_test_type hadoopcheck + +function hadoopcheck_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + add_test hadoopcheck + fi +} + +function hadoopcheck_rebuild +{ + local repostatus=$1 + local hadoopver + local logfile + local count + local result=0 + + if [[ "${repostatus}" = branch ]]; then + return 0 + fi + + big_console_header "Compiling against various Hadoop versions" + + export MAVEN_OPTS="${MAVEN_OPTS}" + for hadoopver in ${HBASE_HADOOP_VERSIONS}; do + logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" + echo_and_redirect "${logfile}" \ + "${MAVEN}" clean install \ + -DskipTests -DHBasePatchProcess \ + -Dhadoop-two.version="${hadoopver}" + count=$(${GREP} -c ERROR "${logfile}") + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${hadoopver}." + ((result=result+1)) + fi + done + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + + add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}." + return 0 +} + +###################################### + +add_test_type hbaseprotoc + +function hbaseprotoc_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.proto$ ]]; then + add_test hbaseprotoc + fi +} + +function hbaseprotoc_rebuild +{ + local i=0 + local fn + local module + local logfile + local count + local result + + if [[ "${repostatus}" = branch ]]; then + return 0 + fi + + verify_needed_test hbaseprotoc + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "Patch HBase protoc plugin" + + start_clock + + + personality_modules patch hbaseprotoc + modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess + + # shellcheck disable=SC2153 + until [[ $i -eq ${#MODULE[@]} ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + module=${MODULE[$i]} + fn=$(module_file_fragment "${module}") + logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt" + + count=$(${GREP} -c ERROR "${logfile}") + + if [[ ${count} -gt 0 ]]; then + module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\ + "${count} new protoc errors in ${module}." + ((result=result+1)) + fi + ((i=i+1)) + done + + modules_messages patch hbaseprotoc true + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +###################################### + +add_test_type hbaseanti + +function hbaseanti_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + add_test hbaseanti + fi +} + +function hbaseanti_patchfile +{ + local patchfile=$1 + local warnings + local result + + verify_needed_test hbaseanti + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "Checking for known anti-patterns" + + start_clock + + warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}") + if [[ ${warnings} -gt 0 ]]; then + add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}." + ((result=result+1)) + fi + + warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}") + if [[ ${warnings} -gt 0 ]]; then + add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}." + ((result=result+1)) + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + + add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns." + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/kafka.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/kafka.sh b/precommit/personality/kafka.sh new file mode 100755 index 0000000..cc4bf09 --- /dev/null +++ b/precommit/personality/kafka.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-checkstyle,-asflicense" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^KAFKA-[0-9]+$' + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="http://kafka.apache.org/contributing.html"; + # shellcheck disable=SC2034 + BUILDTOOL=gradle + #shellcheck disable=SC2034 + GITHUB_REPO="apache/kafka" +} + +function personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare module + declare extra="" + + yetus_debug "Using kafka personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + gradleboot) + # kafka's bootstrap is broken + if [[ ${testtype} == gradleboot ]]; then + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/kafka-configure-gradle.txt" gradle + popd >/dev/null + fi + ;; + compile) + extra="clean jar" + ;; + esac + + for module in ${CHANGED_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/pig.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/pig.sh b/precommit/personality/pig.sh new file mode 100755 index 0000000..50ea52f --- /dev/null +++ b/precommit/personality/pig.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^PIG-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/pig" + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="" + #shellcheck disable=SC2034 + BUILDTOOL=ant +} + +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + local extra="" + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + extra="-DPigPatchProcess= " + + case ${testtype} in + findbugs) + # shellcheck disable=SC2034 + ANT_FINDBUGSXML="${BASEDIR}/build/test/findbugs/pig-findbugs-report.xml" + extra="-Dfindbugs.home=${FINDBUGS_HOME}" + ;; + compile) + extra="${extra} -Djavac.args=-Xlint -Dcompile.c++=yes clean piggybank" + ;; + javadoc) + extra="${extra} -Dforrest.home=${FORREST_HOME}" + ;; + unit) + extra="${extra} -Dtest.junit.output.format=xml -Dcompile.c++=yes -Dtest.output=yes test-core" + ;; + esac + + # shellcheck disable=SC2086 + personality_enqueue_module . ${extra} +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/samza.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/samza.sh b/precommit/personality/samza.sh new file mode 100755 index 0000000..131e73e --- /dev/null +++ b/precommit/personality/samza.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +personality_plugins "all,-checkstyle" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^SAMZA-[0-9]+$' + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/SAMZA/Contributor's+Corner" + # shellcheck disable=SC2034 + BUILDTOOL=gradle + #shellcheck disable=SC2034 + GITHUB_REPO="apache/samza" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/tajo.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/tajo.sh b/precommit/personality/tajo.sh new file mode 100755 index 0000000..aa5fc48 --- /dev/null +++ b/precommit/personality/tajo.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^TAJO-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/tajo" + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/personality/tez.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/tez.sh b/precommit/personality/tez.sh new file mode 100755 index 0000000..584865e --- /dev/null +++ b/precommit/personality/tez.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^TEZ-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/tez" + #shellcheck disable=SC2034 + HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/smart-apply-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/smart-apply-patch.sh b/precommit/smart-apply-patch.sh new file mode 100755 index 0000000..ee06030 --- /dev/null +++ b/precommit/smart-apply-patch.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make sure that bash version meets the pre-requisite + +if [[ -z "${BASH_VERSINFO}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + +this="${BASH_SOURCE-$0}" +BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +#shellcheck disable=SC2034 +QATESTMODE=false + +# dummy functions +function add_vote_table +{ + true +} + +function add_footer_table +{ + true +} + +function big_console_header +{ + true +} + +function add_test +{ + true +} + +## @description Clean the filesystem as appropriate and then exit +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function cleanup_and_exit +{ + local result=$1 + + if [[ ${PATCH_DIR} =~ ^/tmp/yetus + && -d ${PATCH_DIR} ]]; then + rm -rf "${PATCH_DIR}" + fi + + # shellcheck disable=SC2086 + exit ${result} +} + +## @description Setup the default global variables +## @audience public +## @stability stable +## @replaceable no +function setup_defaults +{ + common_defaults +} + +## @description Print the usage information +## @audience public +## @stability stable +## @replaceable no +function yetus_usage +{ + echo "Usage: smart-apply-patch.sh [options] patch" + echo + echo "--committer Apply patches like a boss." + echo "--debug If set, then output some extra stuff to stderr" + echo "--dry-run Check for patch viability without applying" + echo "--list-plugins List all installed plug-ins and then exit" + echo "--modulelist=<list> Specify additional modules to test (comma delimited)" + echo "--offline Avoid connecting to the Internet" + echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/yetus-(random))" + echo "--personality=<file> The personality file to load" + echo "--plugins=<list> Specify which plug-ins to add/delete (comma delimited; use 'all' for all found)" + echo "--project=<name> The short name for project currently using test-patch (default 'yetus')" + echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d" + echo "--user-plugins=<dir> A directory of user provided plugins. see test-patch.d for examples (default empty)" + echo "" + echo "Shell binary overrides:" + echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')" + echo "--curl-cmd=<cmd> The 'curl' command to use (default 'curl')" + echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')" + echo "--file-cmd=<cmd> The 'file' command to use (default 'file')" + echo "--git-cmd=<cmd> The 'git' command to use (default 'git')" + echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')" + echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')" + echo "--sed-cmd=<cmd> The 'sed' command to use (default 'sed')" + + importplugins + + unset TESTFORMATS + unset TESTTYPES + unset BUILDTOOLS + + for plugin in ${BUGSYSTEMS}; do + if declare -f ${plugin}_usage >/dev/null 2>&1; then + echo + "${plugin}_usage" + fi + done +} + +## @description Interpret the command line parameters +## @audience private +## @stability stable +## @replaceable no +## @params $@ +## @return May exit on failure +function parse_args +{ + local i + + common_args "$@" + + for i in "$@"; do + case ${i} in + --committer) + COMMITMODE=true + ;; + --dry-run) + PATCH_DRYRUNMODE=true + ;; + --*) + ## PATCH_OR_ISSUE can't be a --. So this is probably + ## a plugin thing. + continue + ;; + *) + PATCH_OR_ISSUE=${i#*=} + ;; + esac + done + + if [[ ! -d ${PATCH_DIR} ]]; then + mkdir -p "${PATCH_DIR}" + if [[ $? != 0 ]] ; then + yetus_error "ERROR: Unable to create ${PATCH_DIR}" + cleanup_and_exit 1 + fi + fi +} + +## @description git am dryrun +## @replaceable no +## @audience private +## @stability evolving +function gitam_dryrun +{ + + # there is no dryrun method for git-am, so just + # use apply instead. + gitapply_dryrun "$@" + + if [[ ${PATCH_METHOD} = "gitapply" ]]; then + PATCH_METHOD="gitam" + fi +} + +## @description git am signoff +## @replaceable no +## @audience private +## @stability evolving +function gitam_apply +{ + declare patchfile=$1 + + echo "Applying the patch:" + yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-am.log" \ + "${GIT}" am --signoff --whitespace=fix "-p${PATCH_LEVEL}" "${patchfile}" + ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-am.log" +} + +## @description import core library routines +## @audience private +## @stability evolving +function import_core +{ + declare filename + + for filename in "${BINDIR}/core.d"/*; do + # shellcheck disable=SC1091 + # shellcheck source=core.d/01-common.sh + . "${filename}" + done +} + +trap "cleanup_and_exit 1" HUP INT QUIT TERM + +import_core + +setup_defaults + +parse_args "$@" + +importplugins +yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" +unset BUILDTOOLS +unset TESTTYPES +unset TESTFORMATS + +parse_args_plugins "$@" + +plugins_initialize + +locate_patch + +if [[ ${COMMITMODE} = true ]]; then + PATCH_METHODS=("gitam" "${PATCH_METHODS[@]}") +fi + +patchfile_dryrun_driver "${PATCH_DIR}/patch" +RESULT=$? + +if [[ ${RESULT} -gt 0 ]]; then + yetus_error "ERROR: Aborting! ${PATCH_OR_ISSUE} cannot be verified." + cleanup_and_exit ${RESULT} +fi + +if [[ ${PATCH_DRYRUNMODE} == false ]]; then + patchfile_apply_driver "${PATCH_DIR}/patch" + RESULT=$? +fi + +if [[ ${COMMITMODE} = true + && ${PATCH_METHOD} != "gitam" ]]; then + yetus_debug "Running git add -A" + git add -A +fi + +cleanup_and_exit ${RESULT} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch-docker/Dockerfile-endstub ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile-endstub b/precommit/test-patch-docker/Dockerfile-endstub new file mode 100644 index 0000000..aa0463e --- /dev/null +++ b/precommit/test-patch-docker/Dockerfile-endstub @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ADD launch-test-patch.sh /testptch/launch-test-patch.sh +RUN chmod a+rx /testptch/launch-test-patch.sh +CMD /testptch/launch-test-patch.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch-docker/Dockerfile-startstub ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile-startstub b/precommit/test-patch-docker/Dockerfile-startstub new file mode 100644 index 0000000..d191285 --- /dev/null +++ b/precommit/test-patch-docker/Dockerfile-startstub @@ -0,0 +1,91 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:trusty + +WORKDIR /root + +###### +# Install common dependencies from packages +###### +RUN apt-get update && apt-get install --no-install-recommends -y \ + git curl ant make maven \ + cmake gcc g++ protobuf-compiler \ + build-essential libtool \ + zlib1g-dev pkg-config libssl-dev \ + snappy libsnappy-dev \ + bzip2 libbz2-dev \ + libjansson-dev \ + fuse libfuse-dev \ + libcurl4-openssl-dev \ + python python2.7 pylint \ + ruby \ + openjdk-7-jdk \ + libperl-critic-perl + +# Fixing the Apache commons / Maven dependency problem under Ubuntu: +# See http://wiki.apache.org/commons/VfsProblems +RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . + +####### +# Oracle Java +####### + +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:webupd8team/java +RUN apt-get update + + +# Auto-accept the Oracle JDK license +RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +RUN apt-get install -y oracle-java7-installer + +# Auto-accept the Oracle JDK license +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +RUN apt-get install -y oracle-java8-installer + +###### +# Install findbugs +###### +RUN mkdir -p /opt/findbugs && \ + curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ + -o /opt/findbugs.tar.gz && \ + tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs +ENV FINDBUGS_HOME /opt/findbugs + +#### +# Install shellcheck +#### +RUN apt-get install -y cabal-install +RUN cabal update && cabal install shellcheck --global + +#### +# Install rubocop +### +RUN gem install rubocop + +#### +# Install ruby-lint +### +RUN gem install ruby-lint + +#### +# Install bats +#### +RUN add-apt-repository -y ppa:duggan/bats +RUN apt-get update +RUN apt-get install -y bats http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch-docker/launch-test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/launch-test-patch.sh b/precommit/test-patch-docker/launch-test-patch.sh new file mode 100755 index 0000000..ac976ad --- /dev/null +++ b/precommit/test-patch-docker/launch-test-patch.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd "${BASEDIR}" + +if [[ -n ${JAVA_HOME} + && ! -d ${JAVA_HOME} ]]; then + echo "JAVA_HOME: ${JAVA_HOME} does not exist. Dockermode: attempting to switch to another." 1>&2 + JAVA_HOME="" +fi + +if [[ -z ${JAVA_HOME} ]]; then + JAVA_HOME=$(find /usr/lib/jvm/ -name "java-*" -type d | tail -1) + export JAVA_HOME +fi + +# Avoid out of memory errors in builds +MAVEN_OPTS=${MAVEN_OPTS:-"-Xms256m -Xmx1g"} +export MAVEN_OPTS + +# strip out --docker param to prevent re-exec again +TESTPATCHMODE=${TESTPATCHMODE/--docker } + + +cd "${BASEDIR}" +PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P) + +cd "${PATCH_DIR}/precommit/" +#shellcheck disable=SC2086 +"${PATCH_DIR}/precommit/test-patch.sh" \ + --reexec \ + --dockermode ${TESTPATCHMODE} \ + --basedir="${BASEDIR}" \ + --patch-dir="${PATCH_DIR}" \ + --java-home="${JAVA_HOME}" \ + --personality="${PATCH_DIR}/precommit/personality/provided.sh" \ + --user-plugins="${PATCH_DIR}/precommit/user-plugins" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch-docker/test-patch-docker.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/test-patch-docker.sh b/precommit/test-patch-docker/test-patch-docker.sh new file mode 100755 index 0000000..8b19502 --- /dev/null +++ b/precommit/test-patch-docker/test-patch-docker.sh @@ -0,0 +1,388 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DID=${RANDOM} + +## @description Print a message to stderr if --debug is turned on +## @audience private +## @stability stable +## @replaceable no +## @param string +function yetus_debug +{ + if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then + echo "[$(date) DEBUG]: $*" 1>&2 + fi +} + +## @description Run docker with some arguments, and +## @description optionally send to debug +## @audience private +## @stability evolving +## @replaceable no +## @param args +function dockercmd +{ + yetus_debug "docker $*" + docker "$@" +} + +## @description Handle command line arguments +## @audience private +## @stability evolving +## @replaceable no +## @param args +function parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --debug) + YETUS_SHELL_SCRIPT_DEBUG=true + ;; + --dockerversion=*) + DOCKER_VERSION=${i#*=} + ;; + --help|-help|-h|help|--h|--\?|-\?|\?) + yetus_usage + exit 0 + ;; + --java-home=*) + JAVA_HOME=${i#*=} + ;; + --patch-dir=*) + PATCH_DIR=${i#*=} + ;; + --project=*) + PROJECT_NAME=${i#*=} + ;; + *) + ;; + esac + done +} + +## @description Stop and delete all defunct containers +## @audience private +## @stability evolving +## @replaceable no +## @param args +function stop_exited_containers +{ + local line + local id + local value + local size + + echo "Docker containers in exit state:" + + dockercmd ps -a | grep Exited + + # stop *all* containers that are in exit state for + # more than > 8 hours + while read line; do + id=$(echo "${line}" | cut -f1 -d' ') + value=$(echo "${line}" | cut -f2 -d' ') + size=$(echo "${line}" | cut -f3 -d' ') + + if [[ ${size} =~ day + || ${size} =~ week + || ${size} =~ month + || ${size} =~ year ]]; then + echo "Removing docker ${id}" + dockercmd rm "${id}" + fi + + if [[ ${size} =~ hours + && ${value} -gt 8 ]]; then + echo "Removing docker ${id}" + dockercmd rm "${id}" + fi + done < <( + dockercmd ps -a \ + | grep Exited \ + | sed -e 's,ago,,g' \ + | awk '{print $1" "$(NF - 2)" "$(NF - 1)}') +} + +## @description Remove all containers that are not +## @description are not running + older than 1 day +## @audience private +## @stability evolving +## @replaceable no +## @param args +function rm_old_containers +{ + local line + local id + local value + local size + + while read line; do + id=$(echo "${line}" | cut -f1 -d, ) + state=$(echo "${line}" | cut -f2 -d, ) + stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) + + # believe it or not, date is not even close to standardized... + if [[ $(uname -s) == Linux ]]; then + + # GNU date + stoptime=$(date -d "${stoptime}" "+%s") + else + + # BSD date + stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s") + fi + + if [[ ${state} == false ]]; then + curtime=$(date "+%s") + ((difftime = curtime - stoptime)) + if [[ ${difftime} -gt 86400 ]]; then + echo "Removing docker ${id}" + dockercmd rm "${id}" + fi + fi + done < <( + # see https://github.com/koalaman/shellcheck/issues/375 + # shellcheck disable=SC2046 + dockercmd inspect \ + -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \ + $(dockercmd ps -qa) 2>/dev/null) +} + +## @description Remove untagged/unused images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function remove_untagged_images +{ + # this way is a bit more compatible with older docker versions + dockercmd images | tail -n +2 | awk '$1 == "<none>" {print $3}' | \ + xargs --no-run-if-empty docker rmi +} + +## @description Remove defunct tagged images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function remove_old_tagged_images +{ + local line + local id + local created + + while read line; do + id=$(echo "${line}" | awk '{print $1}') + created=$(echo "${line}" | awk '{print $5}') + + if [[ ${created} =~ week + || ${created} =~ month + || ${created} =~ year ]]; then + echo "Removing docker image ${id}" + dockercmd rmi "${id}" + fi + + if [[ ${id} =~ test-patch-base-${PROJECT_NAME}-date ]]; then + if [[ ${created} =~ day + || ${created} =~ hours ]]; then + echo "Removing docker image ${id}" + dockercmd rmi "${id}" + fi + fi + done < <(dockercmd images) + +} + +## @description Performance docker maintenance on Jenkins +## @audience private +## @stability evolving +## @replaceable no +## @param args +function cleanup_apache_jenkins_docker +{ + echo "==========================" + echo "Docker Images:" + dockercmd images + echo "==========================" + echo "Docker Containers:" + dockercmd ps -a + echo "==========================" + + stop_exited_containers + + rm_old_containers + + remove_untagged_images + + remove_old_tagged_images +} + +## @description Clean up our old images used for patch testing +## @audience private +## @stability evolving +## @replaceable no +## @param args +function cleanup_test_patch_images +{ + local images + local imagecount + local rmimage + local rmi + + # we always want to leave at least one of our images + # so that the whole thing doesn't have to be rebuilt. + # This also let's us purge any old images so that + # we can get fresh stuff sometimes + images=$(dockercmd images | grep --color=none "test-patch-tp-${PROJECT_NAME}" | awk '{print $1}') 2>&1 + + # shellcheck disable=SC2086 + imagecount=$(echo ${images} | tr ' ' '\n' | wc -l) + ((imagecount = imagecount - 1 )) + + # shellcheck disable=SC2086 + rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount}) + for rmi in ${rmimage} + do + echo "Removing image ${rmi}" + dockercmd rmi "${rmi}" + done +} + +## @description Perform pre-run maintenance to free up +## @description resources. With --jenkins, it is a lot +## @description more destructive. +## @audience private +## @stability evolving +## @replaceable no +## @param args +function cleanup +{ + if [[ ${TESTPATCHMODE} =~ jenkins ]]; then + cleanup_apache_jenkins_docker + fi + + cleanup_test_patch_images +} + +## @description Deterine the user name and user id of the user +## @description that the docker container should use +## @audience private +## @stability evolving +## @replaceable no +## @param args +function determine_user +{ + # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some + # ... person ... sets it to an account that doesn't actually exist. + # so instead, we need to try and override it with something that's + # probably close to reality. + if [[ ${TESTPATCHMODE} =~ jenkins ]]; then + USER=$(id | cut -f2 -d\( | cut -f1 -d\)) + fi + + if [[ "$(uname -s)" == "Linux" ]]; then + USER_NAME=${SUDO_USER:=$USER} + USER_ID=$(id -u "${USER_NAME}") + GROUP_ID=$(id -g "${USER_NAME}") + else # boot2docker uid and gid + USER_NAME=${USER} + USER_ID=1000 + GROUP_ID=50 + fi +} + +## @description Determine the revision of a dockerfile +## @audience private +## @stability evolving +## @replaceable no +## @param args +function getdockerfilerev +{ + grep 'TEST_PATCH_PRIVATE: gitrev=' \ + "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ + | cut -f2 -d= +} + +## @description Start a test patch docker container +## @audience private +## @stability evolving +## @replaceable no +## @param args +function run_image +{ + local dockerfilerev + local baseimagename + + dockerfilerev=$(getdockerfilerev) + + baseimagename="test-patch-base-${PROJECT_NAME}-${dockerfilerev}" + + # make a base image, if it isn't available + dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker" + + # using the base image, make one that is patch specific + dockercmd build -t "test-patch-tp-${PROJECT_NAME}-${DID}" - <<PatchSpecificDocker +FROM ${baseimagename} +RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} +RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} +RUN chown -R ${USER_NAME} /home/${USER_NAME} +ENV HOME /home/${USER_NAME} +USER ${USER_NAME} +PatchSpecificDocker + + if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then + extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt") + else + extraargs="" + fi + + if [[ ${PATCH_DIR} =~ ^/ ]]; then + dockercmd run --rm=true -i \ + ${extraargs} \ + -v "${PWD}:/testptch/${PROJECT_NAME}" \ + -v "${PATCH_DIR}:/testptch/patchprocess" \ + -u "${USER_NAME}" \ + -w "/testptch/${PROJECT_NAME}" \ + --env=BASEDIR="/testptch/${PROJECT_NAME}" \ + --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ + --env=JAVA_HOME="${JAVA_HOME}" \ + --env=PATCH_DIR=/testptch/patchprocess \ + --env=PROJECT_NAME="${PROJECT_NAME}" \ + --env=TESTPATCHMODE="${TESTPATCHMODE}" \ + "test-patch-tp-${PROJECT_NAME}-${DID}" + else + dockercmd run --rm=true -i \ + ${extraargs} \ + -v "${PWD}:/testptch/${PROJECT_NAME}" \ + -u "${USER_NAME}" \ + -w "/testptch/${PROJECT_NAME}" \ + --env=BASEDIR="/testptch/${PROJECT_NAME}" \ + --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ + --env=JAVA_HOME="${JAVA_HOME}" \ + --env=PATCH_DIR="${PATCH_DIR}" \ + --env=PROJECT_NAME="${PROJECT_NAME}" \ + --env=TESTPATCHMODE="${TESTPATCHMODE}" \ + "test-patch-tp-${PROJECT_NAME}-${DID}" + fi +} + +parse_args "$@" +cleanup +determine_user +run_image http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/ant.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/ant.sh b/precommit/test-patch.d/ant.sh new file mode 100755 index 0000000..9d931a5 --- /dev/null +++ b/precommit/test-patch.d/ant.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "${ANT_HOME:-}" ]]; then + ANT=ant +else + ANT=${ANT_HOME}/bin/ant +fi + +add_build_tool ant + +declare -a ANT_ARGS=("-noinput") + +function ant_usage +{ + echo "ant specific:" + echo "--ant-cmd=<cmd> The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')" +} + +function ant_parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --ant-cmd=*) + ANT=${i#*=} + ;; + esac + done + + # if we requested offline, pass that to ant + if [[ ${OFFLINE} == "true" ]]; then + ANT_ARGS=("${ANT_ARGS[@]}" -Doffline=) + fi +} + +function ant_initialize +{ + # we need to do this before docker kicks in + if [[ -e "${HOME}/.ivy2" + && ! -d "${HOME}/.ivy2" ]]; then + yetus_error "ERROR: ${HOME}/.ivy2 is not a directory." + return 1 + elif [[ ! -e "${HOME}/.ivy2" ]]; then + yetus_debug "Creating ${HOME}/.ivy2" + mkdir -p "${HOME}/.ivy2" + fi +} + +function ant_buildfile +{ + echo "build.xml" +} + +function ant_executor +{ + echo "${ANT}" "${ANT_ARGS[@]}" +} + +function ant_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + shift 2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + findbugs) + modules_workers "${repostatus}" findbugs findbugs + ;; + compile) + modules_workers "${repostatus}" compile + ;; + distclean) + modules_workers "${repostatus}" distclean clean + ;; + javadoc) + modules_workers "${repostatus}" javadoc clean javadoc + ;; + unit) + modules_workers "${repostatus}" unit + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +function ant_javac_count_probs +{ + declare warningfile=$1 + declare val1 + declare val2 + + #shellcheck disable=SC2016 + val1=$(${GREP} -E "\[javac\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + #shellcheck disable=SC2016 + val2=$(${GREP} -E "\[javac\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + echo $((val1+val2)) +} + +## @description Helper for check_patch_javadoc +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function ant_javadoc_count_probs +{ + local warningfile=$1 + local val1 + local val2 + + #shellcheck disable=SC2016 + val1=$(${GREP} -E "\[javadoc\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + #shellcheck disable=SC2016 + val2=$(${GREP} -E "\[javadoc\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}') + echo $((val1+val2)) +} + +function ant_builtin_personality_modules +{ + local repostatus=$1 + local testtype=$2 + + local module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + for module in ${CHANGED_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} + done +} + +function ant_builtin_personality_file_tests +{ + local filename=$1 + + yetus_debug "Using builtin ant personality_file_tests" + + if [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + ]]; then + yetus_debug "tests/shell: ${filename}" + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ src/test + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test javac + add_test unit + elif [[ ${filename} =~ build.xml + || ${filename} =~ ivy.xml + || ${filename} =~ \.java$ + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test javac + add_test javadoc + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +function ant_docker_support +{ + echo "-v ${HOME}/.ivy2:${HOME}/.ivy2" > "${PATCH_DIR}/buildtool-docker-params.txt" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6f38afa5/precommit/test-patch.d/asflicense.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/asflicense.sh b/precommit/test-patch.d/asflicense.sh new file mode 100755 index 0000000..befe7a1 --- /dev/null +++ b/precommit/test-patch.d/asflicense.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +add_test_type asflicense +add_test asflicense + +function asflicense_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --asflicense-rat-excludes=*) + ASFLICENSE_RAT_EXCLUDES=${i#*=} + ;; + --asflicense-rat-jar=*) + ASFLICENSE_RAT_JAR=${i#*=} + ;; + esac + done + + case ${BUILDTOOL} in + ant|gradle|maven) + add_test asflicense + ;; + *) + if [[ -f "${ASFLICENSE_RAT_JAR}" ]]; then + add_test asflicense + fi + ;; + esac +} + +## @description Verify all files have an Apache License +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function asflicense_tests +{ + local numpatch + local btfails=true + local asfex + + big_console_header "Determining number of patched ASF License errors" + + start_clock + + personality_modules patch asflicense + case ${BUILDTOOL} in + ant) + modules_workers patch asflicense releaseaudit + ;; + gradle) + btfails=false + modules_workers patch asflicense rat + ;; + maven) + modules_workers patch asflicense apache-rat:check + ;; + *) + if [[ -z "${ASFLICENSE_RAT_JAR}" ]]; then + return 0 + fi + + if [[ -f ${ASFLICENSE_RAT_EXCLUDES} ]]; then + asfex="-E ${ASFLICENSE_RAT_EXCLUDES} -d ${BASEDIR}" + else + asfex="${BASEDIR}" + fi + + asflicense_writexsl "${PATCH_DIR}/asf.xsl" + echo_and_redirect "${PATCH_DIR}/patch-asflicense.txt" \ + "${JAVA_HOME}/bin/java" \ + -jar "${ASFLICENSE_RAT_JAR}" \ + -s "${PATCH_DIR}/asf.xsl" \ + "${asfex}" + ;; + esac + + # RAT fails the build if there are license problems. + # so let's take advantage of that a bit. + if [[ $? == 0 && ${btfails} = true ]]; then + add_vote_table 1 asflicense "Patch does not generate ASF License warnings." + return 0 + fi + + if [[ ! -f "${PATCH_DIR}/patch-asflicense.txt" ]]; then + #shellcheck disable=SC2038 + find "${BASEDIR}" -name rat.txt \ + -o -name releaseaudit_report.txt \ + -o -name rat-report.txt \ + | xargs cat > "${PATCH_DIR}/patch-asflicense.txt" + fi + + if [[ ! -s "${PATCH_DIR}/patch-asflicense.txt" ]]; then + if [[ ${btfails} = true ]]; then + # if we're here, then build actually failed + modules_messages patch asflicense true + else + add_vote_table 0 asflicense "ASF License check generated no output?" + return 0 + fi + fi + + numpatch=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patch-asflicense.txt") + echo "" + echo "" + echo "There appear to be ${numpatch} ASF License warnings after applying the patch." + if [[ -n ${numpatch} + && ${numpatch} -gt 0 ]] ; then + add_vote_table -1 asflicense "Patch generated ${numpatch} ASF License warnings." + + echo "Lines that start with ????? in the ASF License "\ + "report indicate files that do not have an Apache license header:" \ + > "${PATCH_DIR}/patch-asflicense-problems.txt" + + ${GREP} '\!?????' "${PATCH_DIR}/patch-asflicense.txt" \ + >> "${PATCH_DIR}/patch-asflicense-problems.txt" + + add_footer_table asflicense "@@BASE@@/patch-asflicense-problems.txt" + return 1 + fi + add_vote_table 1 asflicense "Patch does not generate ASF License warnings." + return 0 +} + +function asflicense_writexsl +{ +cat > "${1}" << EOF +<?xml version='1.0' ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one * + or more contributor license agreements. See the NOTICE file * + distributed with this work for additional information * + regarding copyright ownership. The ASF licenses this file * + to you under the Apache License, Version 2.0 (the * + "License"); you may not use this file except in compliance * + with the License. You may obtain a copy of the License at * + * + http://www.apache.org/licenses/LICENSE-2.0 * + * + Unless required by applicable law or agreed to in writing, * + software distributed under the License is distributed on an * + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + KIND, either express or implied. See the License for the * + specific language governing permissions and limitations * + under the License. * +--> +<xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform";> +<xsl:output method='text'/> +<xsl:template match='/'> + Files with Apache License headers will be marked AL + Binary files (which do not require any license headers) will be marked B + Compressed archives will be marked A + Notices, licenses etc. will be marked N + + <xsl:for-each select='descendant::resource'> + <xsl:choose> + <xsl:when test='license-approval/@name="false"'>!</xsl:when> + <xsl:otherwise><xsl:text> </xsl:text></xsl:otherwise> + </xsl:choose> + <xsl:choose> + <xsl:when test='type/@name="notice"'>N </xsl:when> + <xsl:when test='type/@name="archive"'>A </xsl:when> + <xsl:when test='type/@name="binary"'>B </xsl:when> + <xsl:when test='type/@name="standard"'><xsl:value-of select='header-type/@name'/></xsl:when> + <xsl:otherwise>!!!!!</xsl:otherwise> + </xsl:choose> + <xsl:text> </xsl:text> + <xsl:value-of select='@name'/> + <xsl:text> + </xsl:text> + </xsl:for-each> +</xsl:template> +</xsl:stylesheet> +EOF +}
