http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/patchfiles.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/core.d/patchfiles.sh b/precommit/src/main/shell/core.d/patchfiles.sh new file mode 100755 index 0000000..90fb280 --- /dev/null +++ b/precommit/src/main/shell/core.d/patchfiles.sh @@ -0,0 +1,342 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description Use curl to download the patch as a last resort +## @audience private +## @stability evolving +## @param patchloc +## @param output +## @return 0 got something +## @return 1 error +function generic_locate_patch +{ + declare input=$1 + declare output=$2 + + if [[ "${OFFLINE}" == true ]]; then + yetus_debug "generic_locate_patch: offline, skipping" + return 1 + fi + + ${CURL} --silent -L \ + --output "${output}" \ + "${input}" + if [[ $? != 0 ]]; then + yetus_debug "generic_locate_patch: failed to download the patch." + return 1 + fi + return 0 +} + +## @description Given a possible patch file, guess if it's a patch file +## @description only using the more intense verify if we really need to +## @audience private +## @stability evolving +## @param path to patch file to test +## @return 0 we think it's a patch file +## @return 1 we think it's not a patch file +function guess_patch_file +{ + declare patch=$1 + declare fileOutput + + if [[ ! -f ${patch} ]]; then + return 1 + fi + + yetus_debug "Trying to guess if ${patch} is a patch file." + fileOutput=$("${FILE}" "${patch}") + if [[ $fileOutput =~ \ diff\ ]]; then + yetus_debug "file magic says it's a diff." + return 0 + fi + + fileOutput=$(head -n 1 "${patch}" | "${GREP}" -E "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$") + if [[ $? == 0 ]]; then + yetus_debug "first line looks like a patch file." + return 0 + fi + + patchfile_dryrun_driver "${patch}" +} + +## @description Given ${PATCH_OR_ISSUE}, determine what type of patch file is in use, +## @description and do the necessary work to place it into ${PATCH_DIR}/patch. +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure, may exit +function locate_patch +{ + declare bugsys + declare patchfile="" + declare gotit=false + + yetus_debug "locate patch" + + if [[ -z "${PATCH_OR_ISSUE}" ]]; then + yetus_error "ERROR: No patch provided." + cleanup_and_exit 1 + fi + + echo "Processing: ${PATCH_OR_ISSUE}" + # it's a declarely provided file + if [[ -f ${PATCH_OR_ISSUE} ]]; then + patchfile="${PATCH_OR_ISSUE}" + PATCH_SYSTEM=generic + if [[ -f "${PATCH_DIR}/patch" ]]; then + "${DIFF}" -q "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" >/dev/null + if [[ $? -eq 1 ]]; then + rm "${PATCH_DIR}/patch" + fi + fi + else + # run through the bug systems. maybe they know? + for bugsys in ${BUGSYSTEMS}; do + if declare -f ${bugsys}_locate_patch >/dev/null 2>&1; then + "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" + if [[ $? == 0 ]]; then + gotit=true + PATCH_SYSTEM=${bugsys} + fi + fi + done + + # ok, none of the bug systems know. let's see how smart we are + if [[ ${gotit} == false ]]; then + generic_locate_patch "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" + if [[ $? != 0 ]]; then + yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}." + cleanup_and_exit 1 + fi + PATCH_SYSTEM=generic + fi + fi + + yetus_debug "Determined patch system to be ${PATCH_SYSTEM}" + + if [[ ! -f "${PATCH_DIR}/patch" + && -f "${patchfile}" ]]; then + cp "${patchfile}" "${PATCH_DIR}/patch" + if [[ $? == 0 ]] ; then + echo "Patch file ${patchfile} copied to ${PATCH_DIR}" + else + yetus_error "ERROR: Could not copy ${patchfile} to ${PATCH_DIR}" + cleanup_and_exit 1 + fi + fi +} + +## @description if patch-level zero, then verify we aren't +## @description just adding files +## @audience public +## @stability stable +## @param log filename +## @replaceable no +## @return $? +function patchfile_verify_zero +{ + declare logfile=$1 + shift + declare dir + declare changed_files1 + declare changed_files2 + declare filename + + # don't return /dev/null + # shellcheck disable=SC2016 + changed_files1=$(${AWK} 'function p(s){if(s!~"^/dev/null"){print s}} + /^diff --git / { p($3); p($4) } + /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u) + + # maybe we interpreted the patch wrong? check the log file + # shellcheck disable=SC2016 + changed_files2=$(${GREP} -E '^[cC]heck' "${logfile}" \ + | ${AWK} '{print $3}' \ + | ${SED} -e 's,\.\.\.$,,g') + + for filename in ${changed_files1} ${changed_files2}; do + + # leading prefix = bad + if [[ ${filename} =~ ^(a|b)/ ]]; then + return 1 + fi + + # touching an existing file is proof enough + # that pl=0 is good + if [[ -f ${filename} ]]; then + return 0 + fi + + dir=$(dirname "${filename}" 2>/dev/null) + if [[ -n ${dir} && -d ${dir} ]]; then + return 0 + fi + done + + # ¯\_(ã)_/¯ - no way for us to know, all new files with no prefix! + yetus_error "WARNING: Patch only adds files; using patch level ${PATCH_LEVEL}" + return 0 +} + +## @description git apply dryrun +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to dryrun +function gitapply_dryrun +{ + declare patchfile=$1 + declare prefixsize=${2:-0} + + while [[ ${prefixsize} -lt 4 + && -z ${PATCH_METHOD} ]]; do + yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \ + "${GIT}" apply --binary -v --check "-p${prefixsize}" "${patchfile}" + if [[ $? == 0 ]]; then + PATCH_LEVEL=${prefixsize} + PATCH_METHOD=gitapply + break + fi + ((prefixsize=prefixsize+1)) + done + + if [[ ${prefixsize} -eq 0 ]]; then + patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log" + if [[ $? != 0 ]]; then + PATCH_METHOD="" + PATCH_LEVEL="" + gitapply_dryrun "${patchfile}" 1 + fi + fi +} + +## @description patch patch dryrun +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to dryrun +function patchcmd_dryrun +{ + declare patchfile=$1 + declare prefixsize=${2:-0} + + while [[ ${prefixsize} -lt 4 + && -z ${PATCH_METHOD} ]]; do + # shellcheck disable=SC2153 + yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \ + "${PATCH}" "-p${prefixsize}" -E --dry-run < "${patchfile}" + if [[ $? == 0 ]]; then + PATCH_LEVEL=${prefixsize} + PATCH_METHOD=patchcmd + break + fi + ((prefixsize=prefixsize+1)) + done + + if [[ ${prefixsize} -eq 0 ]]; then + patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log" + if [[ $? != 0 ]]; then + PATCH_METHOD="" + PATCH_LEVEL="" + patchcmd_dryrun "${patchfile}" 1 + fi + fi +} + +## @description driver for dryrun methods +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to dryrun +function patchfile_dryrun_driver +{ + declare patchfile=$1 + declare method + + #shellcheck disable=SC2153 + for method in "${PATCH_METHODS[@]}"; do + if declare -f ${method}_dryrun >/dev/null; then + "${method}_dryrun" "${patchfile}" + fi + if [[ -n ${PATCH_METHOD} ]]; then + break + fi + done + + if [[ -n ${PATCH_METHOD} ]]; then + return 0 + fi + return 1 +} + +## @description git patch apply +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to apply +function gitapply_apply +{ + declare patchfile=$1 + declare extraopts + + if [[ "${COMMITMODE}" = true ]]; then + extraopts="--whitespace=fix" + fi + + echo "Applying the patch:" + yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-apply.log" \ + "${GIT}" apply --binary ${extraopts} -v --stat --apply "-p${PATCH_LEVEL}" "${patchfile}" + ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-apply.log" +} + +## @description patch patch apply +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to apply +function patchcmd_apply +{ + declare patchfile=$1 + + echo "Applying the patch:" + yetus_run_and_redirect "${PATCH_DIR}/apply-patch-patch-apply.log" \ + "${PATCH}" "-p${PATCH_LEVEL}" -E < "${patchfile}" + cat "${PATCH_DIR}/apply-patch-patch-apply.log" +} + +## @description driver for patch apply methods +## @replaceable no +## @audience private +## @stability evolving +## @param path to patch file to apply +function patchfile_apply_driver +{ + declare patchfile=$1 + declare gpg=$2 + + if declare -f ${PATCH_METHOD}_apply >/dev/null; then + "${PATCH_METHOD}_apply" "${patchfile}" "${gpg}" + if [[ $? -gt 0 ]]; then + return 1 + fi + else + yetus_error "ERROR: Patching method ${PATCH_METHOD} does not have a way to apply patches!" + return 1 + fi + return 0 +}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/reaper.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/core.d/reaper.sh b/precommit/src/main/shell/core.d/reaper.sh new file mode 100755 index 0000000..ce48c25 --- /dev/null +++ b/precommit/src/main/shell/core.d/reaper.sh @@ -0,0 +1,252 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REAPER_MODE=off # off, report, kill +declare -i REAPER_TOTAL_COUNT=0 +REAPER_DOCKER_ONLY=true +REAPER_ZOMBIE_MODULES=() +REAPER_ZOMBIE_LOGS=() +declare -a REAPER_NAMES + + +## @description Add a regex to the reaper's checklist +## @description NOTE: Users WILL override anything added before +## @description argument parsing! +## @stability evolving +## @audience public +## @replaceable no +function reaper_add_name +{ + yetus_add_array_element REAPER_NAMES "$1" +} + +## @description Reaper-specific usage +## @stability stable +## @audience private +## @replaceable no +function reaper_usage +{ + yetus_add_option "--reapermode={off,report,kill}" "Set unit test reaper mode (default: '${REAPER_MODE}')" + yetus_add_option "--reaperdockeronly=<bool>" "Only run the reaper in --docker (default: ${REAPER_DOCKER_ONLY})" + yetus_add_option "--reapernames=<list>" "List of regexs to search (default build tool dependent)" +} + +## @description Reaper-specific argument parsing +## @stability stable +## @audience private +## @replaceable no +## @param arguments +function reaper_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --reapermode=*) + REAPER_MODE=${i#*=} + ;; + --reaperdockeronly=*) + REAPER_DOCKER_ONLY=${i#*=} + ;; + --reapernames=*) + yetus_comma_to_array REAPER_NAMES "${i#*=}" + ;; + esac + done + + # Running the reaper outside of Dockermode is very dangerous + + if [[ "${REAPER_DOCKER_ONLY}" = "true" && ${DOCKERMODE} = "false" ]]; then + REAPER_MODE="off" + return + fi + + # make sure REAPER_MODE is something valid and turn us on + # as necessary + if [[ "${REAPER_MODE}" = "report" || "${REAPER_MODE}" = "kill" ]]; then + add_test_format reaper + yetus_add_entry EXEC_MODES Reaper + else + REAPER_MODE="off" + fi + +} + +## @description Initialize the reaper +## @stability stable +## @audience private +## @replaceable yes +## @param arguments +function reaper_initialize +{ + determine_user +} + + + +## @description Reaper coprocessor function that +## @description runs outside the law +## @stability evolving +## @audience private +## @replaceable yes +function reaper_coproc_func +{ + declare line + declare i + declare module + declare filefrag + declare cmd + declare args + declare pid + declare -a pidlist + declare -i count + + echo "Reaper watching for: ${REAPER_NAMES[*]}" >> "${PATCH_DIR}/reaper.txt" + + while true; do + read -r cmd + case ${cmd} in + reap) + + read -r module + read -r logfile + + while read -r line; do + ((count=count+1)) + for i in "${REAPER_NAMES[@]}"; do + echo "${line}" | ${GREP} -E "${i}" >> "${PATCH_DIR}/${logfile}" + done + done < <(ps -u "${USER_ID}" -o pid= -o args=) + + pidlist=() + count=0 + while read -r line; do + ((count=count+1)) + pid=$(echo "${line}" | cut -f1 -d' ') + args=$(echo "${line}" | cut -f2- -d' ') + if [[ "${REAPER_MODE}" = "kill" ]]; then + pidlist+=("${pid}") + echo "Killing ${pid} ${args}" >> "${PATCH_DIR}/reaper.txt" 2>&1 + fi + done < <(cat "${PATCH_DIR}/${logfile}") + + # tell our parent how many + # doing this now means killing in the background + echo ${count} + + if [[ ${count} -eq 0 ]]; then + rm "${PATCH_DIR}/${logfile}" + fi + + for i in "${pidlist[@]}"; do + if [[ "${REAPER_MODE}" = "kill" ]]; then + pid_kill "${i}" >> "${PATCH_DIR}/reaper.txt" 2>&1 + fi + done + ;; + exit) + exit 0 + ;; + esac + done +} + +## @description Run the reaper +## @stability evolving +## @audience private +## @replaceable yes +## @param module +## @param testlog +## @param testfrag +function reaper_post_exec +{ + declare module=$1 + declare filefrag=$2 + declare count + declare myfile="${filefrag}-reaper.txt" + declare killmsg="" + + case "${REAPER_MODE}" in + off) + return 0 + ;; + kill) + killmsg=" and killed" + ;; + esac + + yetus_debug "Checking for unreaped processes:" + + # give some time for things to die naturally + sleep 2 + + #shellcheck disable=SC2154,SC2086 + printf "reap\n%s\n%s\n" "${module}" "${myfile}" >&${reaper_coproc[1]} + + #shellcheck disable=SC2154,SC2086 + read -r count <&${reaper_coproc[0]} + + if [[ ${count} -gt 0 ]]; then + ((REAPER_TOTAL_COUNT=REAPER_TOTAL_COUNT+count)) + printf "\nFound%s %s left over processes\n\n" "${killmsg}" "${count}" + REAPER_ZOMBIE_MODULES+=("${module}:${count}") + REAPER_ZOMBIE_LOGS+=("@@BASE@@/${myfile}") + return 1 + fi + + return 0 +} + +## @description Reaper output to the user +## @stability evolving +## @audience private +## @replaceable yes +## @param jdkname +function reaper_finalize_results +{ + declare jdk=$1 + declare fn + + if [[ "${REAPER_MODE}" = "off" ]]; then + return 0 + fi + + if [[ ${#REAPER_ZOMBIE_MODULES[@]} -gt 0 ]] ; then + populate_test_table "${jdk}Unreaped Processes" "${REAPER_ZOMBIE_MODULES[@]}" + for fn in "${REAPER_ZOMBIE_LOGS[@]}"; do + add_footer_table "Unreaped Processes Log" "${fn}" + done + REAPER_ZOMBIE_MODULES=() + REAPER_ZOMBIE_LOGS=() + fi +} + +## @description Reaper output to the user +## @stability evolving +## @audience private +## @replaceable yes +## @param jdkname +function reaper_total_count +{ + + if [[ "${REAPER_MODE}" = "off" ]]; then + return 0 + fi + + if [[ ${REAPER_TOTAL_COUNT} -gt 0 ]]; then + add_vote_table -0 reaper "Unreaped process count: ${REAPER_TOTAL_COUNT}" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/docker-cleanup.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/docker-cleanup.sh b/precommit/src/main/shell/docker-cleanup.sh new file mode 100755 index 0000000..1d17b1b --- /dev/null +++ b/precommit/src/main/shell/docker-cleanup.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no shelldocs required from this file +# SHELLDOC-IGNORE + +# Make sure that bash version meets the pre-requisite + +if [[ -z "${BASH_VERSINFO[0]}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + +this="${BASH_SOURCE-$0}" +BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +BINNAME=${this##*/} +BINNAME=${BINNAME%.sh} +#shellcheck disable=SC2034 +STARTINGDIR=$(pwd) +#shellcheck disable=SC2034 +USER_PARAMS=("$@") +#shellcheck disable=SC2034 +QATESTMODE=false + +# dummy functions +function add_vote_table +{ + true +} + +function add_footer_table +{ + true +} + +function bugsystem_finalreport +{ + true +} + +## @description import core library routines +## @audience private +## @stability evolving +function import_core +{ + declare filename + + for filename in "${BINDIR}/core.d"/*; do + # shellcheck disable=SC1091 + # shellcheck source=core.d/01-common.sh + . "${filename}" + done +} + +## @description import plugins then remove the stuff we don't need +## @audience public +## @stability stable +## @replaceable no +function import_and_clean +{ + importplugins + yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" + unset BUILDTOOLS + unset TESTTYPES + unset TESTFORMATS + + #shellcheck disable=SC2034 + DOCKER_CLEANUP_CMD=true + #shellcheck disable=SC2034 + DOCKERSUPPORT=true + #shellcheck disable=SC2034 + ROBOT=true + #shellcheck disable=SC2034 + DOCKERFAIL="fail" +} + +## @description Clean the filesystem as appropriate and then exit +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function cleanup_and_exit +{ + local result=$1 + + if [[ ${PATCH_DIR} =~ ^/tmp/yetus + && -d ${PATCH_DIR} ]]; then + rm -rf "${PATCH_DIR}" + fi + + # shellcheck disable=SC2086 + exit ${result} +} + +## @description Setup the default global variables +## @audience public +## @stability stable +## @replaceable no +function setup_defaults +{ + common_defaults +} + +## @description Interpret the command line parameters +## @audience private +## @stability stable +## @replaceable no +## @param $@ +## @return May exit on failure +function parse_args +{ + declare i + + common_args "$@" + + for i in "$@"; do + case ${i} in + --robot) + # shellcheck disable=SC2034 + ROBOT=true + ;; + --sentinel) + # shellcheck disable=SC2034 + ROBOT=true + # shellcheck disable=SC2034 + SENTINEL=true + yetus_add_entry EXEC_MODES Sentinel + ;; + esac + done + + docker_parse_args "$@" +} + +## @description Print the usage information +## @audience public +## @stability stable +## @replaceable no +function yetus_usage +{ + import_and_clean + + echo "${BINNAME} [OPTIONS]" + + yetus_add_option "--debug" "If set, then output some extra stuff to stderr" + yetus_add_option "--robot" "Assume this is an automated run" + yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)" + docker_usage + + yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" + yetus_reset_usage +} + +## @description Large display for the user console +## @audience public +## @stability stable +## @replaceable no +## @param string +## @return large chunk of text +function big_console_header +{ + local text="$*" + local spacing=$(( (75+${#text}) /2 )) + printf "\n\n" + echo "============================================================================" + echo "============================================================================" + printf "%*s\n" ${spacing} "${text}" + echo "============================================================================" + echo "============================================================================" + printf "\n\n" +} + +trap "cleanup_and_exit 1" HUP INT QUIT TERM + +import_core + +setup_defaults + +parse_args "$@" + +import_and_clean + +parse_args_plugins "$@" + +docker_initialize +plugins_initialize + +docker_cleanup +RESULT=$? + +cleanup_and_exit ${RESULT} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/it/it-tp.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/it/it-tp.sh b/precommit/src/main/shell/it/it-tp.sh new file mode 100755 index 0000000..0e275f4 --- /dev/null +++ b/precommit/src/main/shell/it/it-tp.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# where to get Apache Yetus +YETUSDIR=${YETUSDIR:-$(pwd)/yetus} + +# project to work with +PROJECT=${PROJECT:-hadoop} + +# temp directory to play in +WORKDIR=${WORKDIR:-/tmp/yetus.${PROJECT}} + +# where to store artifacts: logs, etc. RELATIVE PATH! +PATCHDIR=out + +# set the real dir where our source is. this is a relative path! +BASEDIR=srcdir + +# dockerfile to use +DOCKERFILE=${DOCKERFILE:-${BASEDIR}/dev-support/docker/Dockerfile} + +# make the directory if it doesn't exist +mkdir -p "${WORKDIR}" + +# build out workdir +pushd "${WORKDIR}" || exit 1 + +# checkout the project's source +if [[ ! -d "${BASEDIR}" ]]; then + git clone "https://github.com/apache/${PROJECT}"; "${BASEDIR}" +fi + +# clean out the out dir +rm -rf "${PATCHDIR}" || true + +# make sure they exist +mkdir -p "${PATCHDIR}" + +# if we abort the run in the middle of git, it will leave a present we +# don't want +if [[ -f "${BASEDIR}/.git/index.lock" ]]; then + rm "${BASEDIR}/.git/index.lock" +fi + +# our 'default' args, in (mostly) alphabetical order + +# rsync these files back into the archive dir +YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") + +# where the source is located +YETUS_ARGS+=("--basedir=${BASEDIR}") + +# want to make sure the output is sane for these +YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt") +YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt") +YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html") + +# run in docker mode +YETUS_ARGS+=("--docker") + +# which Dockerfile to use +YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + +# force JDK to be OpenJDK 8 +YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") + +# temp storage, etc +YETUS_ARGS+=("--patch-dir=${PATCHDIR}") + +# plugins to enable. modify as necessary based upon what is being tested +YETUS_ARGS+=("--plugins=jira,maven,briefreport,htmlout") + +# Many projects need a high process limit +YETUS_ARGS+=("--proclimit=5000") + +# project name. this will auto trigger personality for built-ins +YETUS_ARGS+=("--project=${PROJECT}") + +# nuke the src repo before working +YETUS_ARGS+=("--resetrepo") + +# run test-patch from the source tree specified up above +TESTPATCHBIN=${YETUSDIR}/precommit/test-patch.sh + +# now run test-patch with any optional arguments: +# --empty-patch for a full run aka 'qbt' +# URL for a remote patch file +# file name for local patch file +# JIRA Issue, etc, etc. +# +# also, can add parameters or override the above as necessary + +cat <<EOF + +******************* +Starting test-patch +******************* + +EOF + +/bin/bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${@}" + +cat <<EOF + +******************* +Stopping test-patch +******************* + +EOF + +popd || exit 1 http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/accumulo.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/accumulo.sh b/precommit/src/main/shell/personality/accumulo.sh new file mode 100755 index 0000000..f2d3e7c --- /dev/null +++ b/precommit/src/main/shell/personality/accumulo.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run all plugins +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^ACCUMULO-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/accumulo" + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="http://accumulo.apache.org/git.html#contributors"; + # We want to invoke the 'check' goal, not the default 'checkstyle'. We define + # our own checkstyle rules which isn't possible via 'checkstyle' configuration + #shellcheck disable=SC2034 + CHECKSTYLE_GOAL="check" +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + + yetus_debug "Personality: ${repostatus} ${testtype}" + clear_personality_queue + + if [[ ${testtype} == 'unit' ]]; then + # Run all tests, not just the tests in the modules affected + yetus_debug "Overriding to run all unit tests" + + personality_enqueue_module . + return + fi + + # Make sure we re-add the changed modules if we didn't short-circuit out + for module in "${CHANGED_MODULES[@]}"; do + personality_enqueue_module "${module}" + done +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/bigtop.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/bigtop.sh b/precommit/src/main/shell/personality/bigtop.sh new file mode 100755 index 0000000..c374289 --- /dev/null +++ b/precommit/src/main/shell/personality/bigtop.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-checkstyle,-findbugs" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + # shellcheck disable=SC2034 + JIRA_ISSUE_RE='^BIGTOP-[0-9]+$' + # shellcheck disable=SC2034 + BUILDTOOL=gradle + # shellcheck disable=SC2034 + GITHUB_REPO="apache/bigtop" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/drill.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/drill.sh b/precommit/src/main/shell/personality/drill.sh new file mode 100755 index 0000000..cc35d45 --- /dev/null +++ b/precommit/src/main/shell/personality/drill.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + GITHUB_REPO="apache/drill" + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://drill.apache.org/docs/apache-drill-contribution-guidelines/"; + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^DRILL-[0-9]+$' + #shellcheck disable=SC2034 + JIRA_STATUS_RE='Reviewable' +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/flink.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/flink.sh b/precommit/src/main/shell/personality/flink.sh new file mode 100755 index 0000000..a9c0fdf --- /dev/null +++ b/precommit/src/main/shell/personality/flink.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^FLINK-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/flink" +} + +add_test_type flinklib + +## @description flinklib file filter +## @audience private +## @stability evolving +## @param filename +function flinklib_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ + || ${filename} =~ \.scala$ + || ${filename} =~ pom.xml$ ]]; then + add_test flinklib + fi +} + +## @description count files in the lib dir +## @audience private +## @stability evolving +function flinklib_count +{ + find "${BASEDIR}" \ + | ${GREP} "/lib/" \ + | ${GREP} -v "_qa_workdir" \ + | wc -l +} + +## @description check fliblib +## @audience private +## @stability evolving +## @param repostatus +function flinklib_rebuild +{ + declare repostatus=$1 + + start_clock + big_console_header "${PATCH_BRANCH} flink library dependencies" + + if ! verify_needed_test flinklib; then + echo "${BUILDMODEMSG} does not need flinklib testing." + return 0 + fi + + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/${repostatus}-flinklib-root.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch + if [[ $? != 0 ]]; then + add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}." + fi + + if [[ ${repostatus} = branch ]]; then + FLINK_PRE_LIB_FILES=$(flinklib_count) + return 0 + else + FLINK_POST_LIB_FILES=$(flinklib_count) + fi + popd >/dev/null + + if [[ "${BUILDMODE}" = full ]]; then + if [[ ${FLINK_POST_LIB_FILES} -gt 0 ]]; then + add_vote_table -1 flinklib "Lib folder dependencies are currently ${FLINK_POST_LIB_FILES}" + return 1 + else + add_vote_table +1 flinklib "No lib folder dependencies!" + return 0 + fi + fi + + if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then + add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \ + "${FLINK_PRE_LIB_FILES} to ${FLINK_POST_LIB_FILES}" + return 1 + elif [[ "${FLINK_POST_LIB_FILES}" -eq "${FLINK_PRE_LIB_FILES}" ]]; then + add_vote_table 0 flinklib "Patch did not change lib dependencies" \ + " (still ${FLINK_PRE_LIB_FILES})" + else + add_vote_table +1 flinklib "Patch decreases lib folder dependencies by " \ + "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))." + fi + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/geode.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/geode.sh b/precommit/src/main/shell/personality/geode.sh new file mode 100755 index 0000000..9f5c0ec --- /dev/null +++ b/precommit/src/main/shell/personality/geode.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-ant,-javac,-scalac,-scaladoc" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=develop + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/GEODE/How+to+Contribute"; + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^(GEODE)-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/incubator-geode" + #shellcheck disable=SC2034 + BUILDTOOL=gradle +# PYLINT_OPTIONS="--indent-string=' '" + +# HADOOP_MODULES="" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/hadoop.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/hadoop.sh b/precommit/src/main/shell/personality/hadoop.sh new file mode 100755 index 0000000..2ad4459 --- /dev/null +++ b/precommit/src/main/shell/personality/hadoop.sh @@ -0,0 +1,563 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# SHELLDOC-IGNORE +# +# Override these to match Apache Hadoop's requirements + +personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://wiki.apache.org/hadoop/HowToContribute"; + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS|HDDS)-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/hadoop" + #shellcheck disable=SC2034 + PYLINT_OPTIONS="--indent-string=' '" + + HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)} + if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then + HADOOP_HOMEBREW_DIR=/usr/local + fi +} + +function personality_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --hadoop-isal-prefix=*) + ISAL_HOME=${i#*=} + ;; + --hadoop-openssl-prefix=*) + OPENSSL_HOME=${i#*=} + ;; + --hadoop-snappy-prefix=*) + SNAPPY_HOME=${i#*=} + ;; + esac + done +} + +## @description Calculate the actual module ordering +## @audience private +## @stability evolving +## @param ordering +function hadoop_order +{ + declare ordering=$1 + declare hadoopm + + if [[ ${ordering} = normal ]]; then + hadoopm="${CHANGED_MODULES[*]}" + elif [[ ${ordering} = union ]]; then + hadoopm="${CHANGED_UNION_MODULES}" + elif [[ ${ordering} = mvnsrc ]]; then + hadoopm="${MAVEN_SRC_MODULES[*]}" + elif [[ ${ordering} = mvnsrctest ]]; then + hadoopm="${MAVEN_SRCTEST_MODULES[*]}" + else + hadoopm="${ordering}" + fi + echo "${hadoopm}" +} + +## @description Determine if it is safe to run parallel tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_test_parallel +{ + if [[ -f "${BASEDIR}/pom.xml" ]]; then + HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \ + | head -1 \ + | "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \ + | cut -f1 -d- ) + export HADOOP_VERSION + else + return 1 + fi + + hmajor=${HADOOP_VERSION%%\.*} + hmajorminor=${HADOOP_VERSION%\.*} + hminor=${hmajorminor##*\.} + # ... and just for reference + #hmicro=${HADOOP_VERSION##*\.} + + # Apache Hadoop v2.8.0 was the first one to really + # get working parallel unit tests + if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then + return 1 + fi + + return 0 +} + +## @description Install extra modules for unit tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_unittest_prereqs +{ + declare input=$1 + declare mods + declare need_common=0 + declare building_common=0 + declare module + declare flags + declare fn + + # prior to running unit tests, hdfs needs libhadoop.so built + # if we're building root, then this extra work is moot + + #shellcheck disable=SC2086 + mods=$(hadoop_order ${input}) + + for module in ${mods}; do + if [[ ${module} = hadoop-hdfs-project* ]]; then + need_common=1 + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then + building_common=1 + elif [[ ${module} = . ]]; then + return + fi + done + + # Windows builds *ALWAYS* need hadoop-common compiled + case ${OSTYPE} in + Windows_NT|CYGWIN*|MINGW*|MSYS*) + need_common=1 + ;; + esac + + if [[ ${need_common} -eq 1 + && ${building_common} -eq 0 ]]; then + echo "unit test pre-reqs:" + module="hadoop-common-project/hadoop-common" + fn=$(module_file_fragment "${module}") + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + pushd "${BASEDIR}/${module}" >/dev/null + # shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} + popd >/dev/null + fi +} + +## @description Calculate the flags/settings for yarn-ui v2 build +## @description based upon the OS +## @audience private +## @stability evolving +function yarn_ui2_flag +{ + + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + # Now it only tested on Linux/OSX, don't enable the profile on + # windows until it get verified + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo -Pyarn-ui + ;; + Darwin) + echo -Pyarn-ui + ;; + *) + # Do nothing + ;; + esac +} + +## @description Calculate the flags/settings for native code +## @description based upon the OS +## @audience private +## @stability evolving +function hadoop_native_flags +{ + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + declare -a args + + # Based upon HADOOP-11937 + # + # Some notes: + # + # - getting fuse to compile on anything but Linux + # is always tricky. + # - Darwin assumes homebrew is in use. + # - HADOOP-12027 required for bzip2 on OS X. + # - bzip2 is broken in lots of places + # (the shared library is considered experimental) + # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 + # + + args=("-Drequire.test.libhadoop") + + if [[ -d "${ISAL_HOME}/include" ]]; then + args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}") + fi + + if [[ -d "${OPENSSL_HOME}/include" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/") + fi + + if [[ -d "${SNAPPY_HOME}/include" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy") + fi + + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo \ + -Pnative \ + -Drequire.fuse \ + -Drequire.openssl \ + -Drequire.snappy \ + -Drequire.valgrind \ + "${args[@]}" + ;; + Darwin) + echo \ + "${args[@]}" \ + -Pnative \ + -Drequire.snappy \ + -Drequire.openssl + ;; + Windows_NT|CYGWIN*|MINGW*|MSYS*) + echo \ + "${args[@]}" \ + -Drequire.snappy -Drequire.openssl -Pnative-win + ;; + *) + echo \ + "${args[@]}" + ;; + esac +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare extra="" + declare ordering="normal" + declare needflags=false + declare foundbats=false + declare flags + declare fn + declare i + declare hadoopm + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + asflicense) + # this is very fast and provides the full path if we do it from + # the root of the source + personality_enqueue_module . + return + ;; + checkstyle) + ordering="union" + extra="-DskipTests" + ;; + compile) + ordering="union" + extra="-DskipTests" + needflags=true + + # if something in common changed, we build the whole world + if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then + yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " + ordering="." + fi + ;; + distclean) + ordering="." + extra="-DskipTests" + ;; + javadoc) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then + echo "javadoc pre-reqs:" + for i in hadoop-project \ + hadoop-common-project/hadoop-annotations; do + fn=$(module_file_fragment "${i}") + pushd "${BASEDIR}/${i}" >/dev/null + echo "cd ${i}" + echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install + popd >/dev/null + done + fi + extra="-Pdocs -DskipTests" + ;; + mvneclipse) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + mvninstall) + extra="-DskipTests" + if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then + ordering=. + fi + ;; + mvnsite) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + unit) + if [[ "${BUILDMODE}" = full ]]; then + ordering=mvnsrc + elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ ${TEST_PARALLEL} = "true" ]] ; then + if hadoop_test_parallel; then + extra="-Pparallel-tests" + if [[ -n ${TEST_THREADS:-} ]]; then + extra="${extra} -DtestsThreadCount=${TEST_THREADS}" + fi + fi + fi + needflags=true + hadoop_unittest_prereqs "${ordering}" + + if ! verify_needed_test javac; then + yetus_debug "hadoop: javac not requested" + if ! verify_needed_test native; then + yetus_debug "hadoop: native not requested" + yetus_debug "hadoop: adding -DskipTests to unit test" + extra="-DskipTests" + fi + fi + + for i in "${CHANGED_FILES[@]}"; do + if [[ "${i}" =~ \.bats ]]; then + foundbats=true + fi + done + + if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then + yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile" + extra="${extra} -P!shelltest" + else + extra="${extra} -Pshelltest" + fi + ;; + *) + extra="-DskipTests" + ;; + esac + + if [[ ${needflags} = true ]]; then + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + extra="${extra} ${flags}" + fi + + extra="-Ptest-patch ${extra}" + + for module in $(hadoop_order ${ordering}); do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +## @description Add tests based upon personality needs +## @audience private +## @stability evolving +## @param filename +function personality_file_tests +{ + declare filename=$1 + + yetus_debug "Using Hadoop-specific personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + add_test shadedclient + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/scripts + || ${filename} =~ src/test/scripts + || ${filename} =~ src/main/bin + || ${filename} =~ shellprofile\.d + || ${filename} =~ src/main/conf + ]]; then + yetus_debug "tests/shell: ${filename}" + add_test mvnsite + add_test unit + elif [[ ${filename} =~ \.md$ + || ${filename} =~ \.md\.vm$ + || ${filename} =~ src/site + ]]; then + yetus_debug "tests/site: ${filename}" + add_test mvnsite + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test cc + add_test mvnsite + add_test javac + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + || ${filename} =~ src/main + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test compile + add_test javac + add_test javadoc + add_test mvninstall + add_test mvnsite + add_test unit + add_test shadedclient + fi + + # if we change anything in here, e.g. the test scripts + # then run the client artifact tests + if [[ ${filename} =~ hadoop-client-modules ]]; then + add_test shadedclient + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests: src/test" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +## @description Image to print on success +## @audience private +## @stability evolving +function hadoop_console_success +{ + printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; + printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; + printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; + printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; + printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; + printf "ICB8bXwgICB8bXwgIAo" +} + +################################################### +# Hadoop project specific check of IT for shaded artifacts + +add_test_type shadedclient + +## @description check for test modules and add test/plugins as needed +## @audience private +## @stability evolving +function shadedclient_initialize +{ + maven_add_install shadedclient +} + +## @description build client facing shaded artifacts and test them +## @audience private +## @stability evolving +## @param repostatus +function shadedclient_rebuild +{ + declare repostatus=$1 + declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt" + declare module + declare -a modules=() + + if [[ ${OSTYPE} = Windows_NT || + ${OSTYPE} =~ ^CYGWIN.* || + ${OSTYPE} =~ ^MINGW32.* || + ${OSTYPE} =~ ^MSYS.* ]]; then + echo "hadoop personality: building on windows, skipping check of client artifacts." + return 0 + fi + + yetus_debug "hadoop personality: seeing if we need the test of client artifacts." + for module in hadoop-client-modules/hadoop-client-check-invariants \ + hadoop-client-modules/hadoop-client-check-test-invariants \ + hadoop-client-modules/hadoop-client-integration-tests; do + if [ -d "${module}" ]; then + yetus_debug "hadoop personality: test module '${module}' is present." + modules=( ${modules[@]} -pl ${module} ) + fi + done + if [ ${#modules[@]} -eq 0 ]; then + echo "hadoop personality: no test modules present, skipping check of client artifacts." + return 0 + fi + + big_console_header "Checking client artifacts on ${repostatus}" + + echo_and_redirect "${logfile}" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ + "${modules[@]}" \ + -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true + + count=$(${GREP} -c '\[ERROR\]' "${logfile}") + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts." + return 1 + fi + + add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts." + return 0 +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/hbase.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/hbase.sh b/precommit/src/main/shell/personality/hbase.sh new file mode 100755 index 0000000..b48691e --- /dev/null +++ b/precommit/src/main/shell/personality/hbase.sh @@ -0,0 +1,368 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + BUILDTOOL=maven + #shellcheck disable=SC2034 + PROJECT_NAME=hbase + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^HBASE-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/hbase" + + # TODO use PATCH_BRANCH to select hadoop versions to use. + # All supported Hadoop versions that we want to test the compilation with + HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1" + + # TODO use PATCH_BRANCH to select jdk versions to use. + + # Override the maven options + MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}" + +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + local extra="" + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + extra="-DHBasePatchProcess" + + if [[ ${repostatus} == branch + && ${testtype} == mvninstall ]] || + [[ "${BUILDMODE}" == full ]];then + personality_enqueue_module . ${extra} + return + fi + + if [[ ${testtype} = findbugs ]]; then + for module in "${CHANGED_MODULES[@]}"; do + # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing + # in src/main/java where findbugs goes to look + if [[ ${module} == hbase-shell ]]; then + continue + elif [[ ${module} == hbase-it ]]; then + continue + else + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + fi + done + return + fi + + if [[ ${testtype} = unit ]]; then + extra="${extra} -PrunAllTests" + + # Inject the jenkins build-id for our surefire invocations + # Used by zombie detection stuff, even though we're not including that yet. + if [ -n "${BUILD_ID}" ]; then + extra="${extra} -Dbuild.id=${BUILD_ID}" + fi + fi + + for module in "${CHANGED_MODULES[@]}"; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +################################################### +# Below here are our one-off tests specific to hbase. +# TODO break them into individual files so it's easier to maintain them? + +# TODO line length check? could ignore all java files since checkstyle gets them. + +################################################### + +add_test_type hadoopcheck + +## @description hadoopcheck file filter +## @audience private +## @stability evolving +## @param filename +function hadoopcheck_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + add_test hadoopcheck + fi +} + +## @description hadoopcheck test +## @audience private +## @stability evolving +## @param repostatus +function hadoopcheck_rebuild +{ + local repostatus=$1 + local hadoopver + local logfile + local count + local result=0 + + if [[ "${repostatus}" = branch ]]; then + return 0 + fi + + big_console_header "Compiling against various Hadoop versions" + + export MAVEN_OPTS="${MAVEN_OPTS}" + for hadoopver in ${HBASE_HADOOP_VERSIONS}; do + logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" + echo_and_redirect "${logfile}" \ + "${MAVEN}" clean install \ + -DskipTests -DHBasePatchProcess \ + -Dhadoop-two.version="${hadoopver}" + count=$(${GREP} -c ERROR "${logfile}") + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}." + ((result=result+1)) + fi + done + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + + add_vote_table +1 hadoopcheck "${BUILDMODEMSG} does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}." + return 0 +} + +###################################### + +# TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol +add_test_type hbaseprotoc + +## @description hbaseprotoc file filter +## @audience private +## @stability evolving +## @param filename +function hbaseprotoc_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.proto$ ]]; then + add_test hbaseprotoc + fi +} + +## @description hadoopcheck test +## @audience private +## @stability evolving +## @param repostatus +function hbaseprotoc_rebuild +{ + declare repostatus=$1 + declare i=0 + declare fn + declare module + declare logfile + declare count + declare result + + if [[ "${repostatus}" = branch ]]; then + return 0 + fi + + if ! verify_needed_test hbaseprotoc; then + return 0 + fi + + big_console_header "HBase protoc plugin: ${BUILDMODE}" + + start_clock + + personality_modules patch hbaseprotoc + modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess + + # shellcheck disable=SC2153 + until [[ $i -eq "${#MODULE[@]}" ]]; do + if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then + ((result=result+1)) + ((i=i+1)) + continue + fi + module=${MODULE[$i]} + fn=$(module_file_fragment "${module}") + logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt" + + count=$(${GREP} -c ERROR "${logfile}") + + if [[ ${count} -gt 0 ]]; then + module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\ + "${count} new protoc errors in ${module}." + ((result=result+1)) + fi + ((i=i+1)) + done + + modules_messages patch hbaseprotoc true + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +###################################### + +add_test_type hbaseanti + +## @description hbaseanti file filter +## @audience private +## @stability evolving +## @param filename +function hbaseanti_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + add_test hbaseanti + fi +} + +## @description hbaseanti patch file check +## @audience private +## @stability evolving +## @param filename +function hbaseanti_patchfile +{ + local patchfile=$1 + local warnings + local result + + if [[ "${BUILDMODE}" = full ]]; then + return 0 + fi + + if ! verify_needed_test hbaseanti; then + return 0 + fi + + big_console_header "Checking for known anti-patterns" + + start_clock + + warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}") + if [[ ${warnings} -gt 0 ]]; then + add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}." + ((result=result+1)) + fi + + warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}") + if [[ ${warnings} -gt 0 ]]; then + add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}." + ((result=result+1)) + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + + add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns." + return 0 +} + + +## @description hbase custom mvnsite file filter. See HBASE-15042 +## @audience private +## @stability evolving +## @param filename +function mvnsite_filefilter +{ + local filename=$1 + + if [[ ${BUILDTOOL} = maven ]]; then + if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then + yetus_debug "tests/mvnsite: ${filename}" + add_test mvnsite + fi + fi +} + +## This is named so that yetus will check us right after running tests. +## Essentially, we check for normal failures and then we look for zombies. +#function hbase_unit_logfilter +#{ +# declare testtype="unit" +# declare input=$1 +# declare output=$2 +# declare processes +# declare process_output +# declare zombies +# declare zombie_count=0 +# declare zombie_process +# +# yetus_debug "in hbase-specific unit logfilter." +# +# # pass-through to whatever is counting actual failures +# if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then +# "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}" +# elif declare -f ${testtype}_logfilter >/dev/null; then +# "${testtype}_logfilter" "${input}" "${output}" +# fi +# +# start_clock +# if [ -n "${BUILD_ID}" ]; then +# yetus_debug "Checking for zombie test processes." +# processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}") +# if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then +# yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop." +# yetus_debug "${processes}" +# sleep 30 +# #shellcheck disable=SC2016 +# for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do +# # Test our zombie still running (and that it still an hbase build item) +# process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}") +# if [[ -n "${process_output}" ]]; then +# yetus_error "Zombie: ${process_output}" +# ((zombie_count = zombie_count + 1)) +# zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3) +# zombies="${zombies} ${zombie_process}" +# fi +# done +# fi +# if [ "${zombie_count}" -ne 0 ]; then +# add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)" +# populate_test_table "zombie unit tests" "${zombies}" +# else +# yetus_info "Zombie check complete. All test runs exited normally." +# stop_clock +# fi +# else +# add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies." +# fi +# +#} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/jmeter.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/jmeter.sh b/precommit/src/main/shell/personality/jmeter.sh new file mode 100755 index 0000000..b4e61b5 --- /dev/null +++ b/precommit/src/main/shell/personality/jmeter.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-javadoc,-findbugs,-asflicense" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + # shellcheck disable=SC2034 + BUILDTOOL=ant + # shellcheck disable=SC2034 + GITHUB_REPO="apache/jmeter" + # shellcheck disable=SC2034 + JMETER_DOWNLOAD_JARS=false +} + +add_test_type jmeter + +## @description Personality usage options +## @audience private +## @stability evolving +function jmeter_usage +{ + yetus_add_option "--jmeter-download-jars=<bool>" "download third-party jars needed by ant build" +} + +## @description Process personality options +## @audience private +## @stability evolving +## @param arguments +function jmeter_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --jmeter-download-jars=*) + JMETER_DOWNLOAD_JARS=${i#*=} + ;; + esac + done +} + +## @description Download jmetere dependencies +## @audience private +## @stability evolving +function jmeter_precheck +{ + if [[ ${JMETER_DOWNLOAD_JARS} = true ]]; then + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/jmeter-branch-download-jars.txt" "${ANT}" download_jars + popd >/dev/null + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/kafka.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/kafka.sh b/precommit/src/main/shell/personality/kafka.sh new file mode 100755 index 0000000..0c2f749 --- /dev/null +++ b/precommit/src/main/shell/personality/kafka.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-checkstyle,-asflicense" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^KAFKA-[0-9]+$' + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="http://kafka.apache.org/contributing.html"; + # shellcheck disable=SC2034 + BUILDTOOL=gradle + #shellcheck disable=SC2034 + GITHUB_REPO="apache/kafka" +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare module + declare extra="" + + yetus_debug "Using kafka personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + gradleboot) + # kafka's bootstrap is broken + if [[ ${testtype} == gradleboot ]]; then + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/kafka-configure-gradle.txt" gradle + popd >/dev/null + fi + ;; + compile) + extra="clean jar" + ;; + esac + + for module in "${CHANGED_MODULES[@]}"; do + # shellcheck disable=SC2086 + personality_enqueue_module "${module}" ${extra} + done +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/orc.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/orc.sh b/precommit/src/main/shell/personality/orc.sh new file mode 100755 index 0000000..0582b17 --- /dev/null +++ b/precommit/src/main/shell/personality/orc.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-checkstyle,-findbugs" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^ORC-[0-9]+$' + # shellcheck disable=SC2034 + BUILDTOOL=cmake + #shellcheck disable=SC2034 + GITHUB_REPO="apache/orc" + + # protobuf fails + #shellcheck disable=SC2034 + CMAKE_ROOT_BUILD=true +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/pig.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/pig.sh b/precommit/src/main/shell/personality/pig.sh new file mode 100755 index 0000000..2014fda --- /dev/null +++ b/precommit/src/main/shell/personality/pig.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^PIG-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/pig" + #shellcheck disable=SC2034 + BUILDTOOL=ant +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + local extra="" + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + extra="-DPigPatchProcess= " + + case ${testtype} in + findbugs) + # shellcheck disable=SC2034 + ANT_FINDBUGSXML="${BASEDIR}/build/test/findbugs/pig-findbugs-report.xml" + extra="-Dfindbugs.home=${FINDBUGS_HOME}" + ;; + compile) + extra="${extra} -Djavac.args=-Xlint -Dcompile.c++=yes clean piggybank" + ;; + javadoc) + extra="${extra} -Dforrest.home=${FORREST_HOME}" + ;; + unit) + extra="${extra} -Dtest.junit.output.format=xml -Dcompile.c++=yes -Dtest.output=yes test-core" + ;; + esac + + # shellcheck disable=SC2086 + personality_enqueue_module . ${extra} +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/ratis.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/ratis.sh b/precommit/src/main/shell/personality/ratis.sh new file mode 100755 index 0000000..d700512 --- /dev/null +++ b/precommit/src/main/shell/personality/ratis.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^RATIS-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/incubator-ratis" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/samza.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/samza.sh b/precommit/src/main/shell/personality/samza.sh new file mode 100755 index 0000000..430ffbb --- /dev/null +++ b/precommit/src/main/shell/personality/samza.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all,-checkstyle" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^SAMZA-[0-9]+$' + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/SAMZA/Contributor's+Corner" + # shellcheck disable=SC2034 + BUILDTOOL=gradle + #shellcheck disable=SC2034 + GITHUB_REPO="apache/samza" +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/tajo.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/tajo.sh b/precommit/src/main/shell/personality/tajo.sh new file mode 100755 index 0000000..fb86ff7 --- /dev/null +++ b/precommit/src/main/shell/personality/tajo.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^TAJO-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/tajo" + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"; +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/tez.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/tez.sh b/precommit/src/main/shell/personality/tez.sh new file mode 100755 index 0000000..32d9ffc --- /dev/null +++ b/precommit/src/main/shell/personality/tez.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^TEZ-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/tez" + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"; +} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/yetus.sh ---------------------------------------------------------------------- diff --git a/precommit/src/main/shell/personality/yetus.sh b/precommit/src/main/shell/personality/yetus.sh new file mode 100755 index 0000000..3afa6d2 --- /dev/null +++ b/precommit/src/main/shell/personality/yetus.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +personality_plugins "all" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^YETUS-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/yetus" +}
