http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/patchfiles.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/patchfiles.sh b/precommit/core.d/patchfiles.sh deleted file mode 100755 index 90fb280..0000000 --- a/precommit/core.d/patchfiles.sh +++ /dev/null @@ -1,342 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description Use curl to download the patch as a last resort -## @audience private -## @stability evolving -## @param patchloc -## @param output -## @return 0 got something -## @return 1 error -function generic_locate_patch -{ - declare input=$1 - declare output=$2 - - if [[ "${OFFLINE}" == true ]]; then - yetus_debug "generic_locate_patch: offline, skipping" - return 1 - fi - - ${CURL} --silent -L \ - --output "${output}" \ - "${input}" - if [[ $? != 0 ]]; then - yetus_debug "generic_locate_patch: failed to download the patch." - return 1 - fi - return 0 -} - -## @description Given a possible patch file, guess if it's a patch file -## @description only using the more intense verify if we really need to -## @audience private -## @stability evolving -## @param path to patch file to test -## @return 0 we think it's a patch file -## @return 1 we think it's not a patch file -function guess_patch_file -{ - declare patch=$1 - declare fileOutput - - if [[ ! -f ${patch} ]]; then - return 1 - fi - - yetus_debug "Trying to guess if ${patch} is a patch file." - fileOutput=$("${FILE}" "${patch}") - if [[ $fileOutput =~ \ diff\ ]]; then - yetus_debug "file magic says it's a diff." - return 0 - fi - - fileOutput=$(head -n 1 "${patch}" | "${GREP}" -E "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$") - if [[ $? == 0 ]]; then - yetus_debug "first line looks like a patch file." - return 0 - fi - - patchfile_dryrun_driver "${patch}" -} - -## @description Given ${PATCH_OR_ISSUE}, determine what type of patch file is in use, -## @description and do the necessary work to place it into ${PATCH_DIR}/patch. -## @audience private -## @stability evolving -## @replaceable no -## @return 0 on success -## @return 1 on failure, may exit -function locate_patch -{ - declare bugsys - declare patchfile="" - declare gotit=false - - yetus_debug "locate patch" - - if [[ -z "${PATCH_OR_ISSUE}" ]]; then - yetus_error "ERROR: No patch provided." - cleanup_and_exit 1 - fi - - echo "Processing: ${PATCH_OR_ISSUE}" - # it's a declarely provided file - if [[ -f ${PATCH_OR_ISSUE} ]]; then - patchfile="${PATCH_OR_ISSUE}" - PATCH_SYSTEM=generic - if [[ -f "${PATCH_DIR}/patch" ]]; then - "${DIFF}" -q "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" >/dev/null - if [[ $? -eq 1 ]]; then - rm "${PATCH_DIR}/patch" - fi - fi - else - # run through the bug systems. maybe they know? - for bugsys in ${BUGSYSTEMS}; do - if declare -f ${bugsys}_locate_patch >/dev/null 2>&1; then - "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" - if [[ $? == 0 ]]; then - gotit=true - PATCH_SYSTEM=${bugsys} - fi - fi - done - - # ok, none of the bug systems know. let's see how smart we are - if [[ ${gotit} == false ]]; then - generic_locate_patch "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" - if [[ $? != 0 ]]; then - yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}." - cleanup_and_exit 1 - fi - PATCH_SYSTEM=generic - fi - fi - - yetus_debug "Determined patch system to be ${PATCH_SYSTEM}" - - if [[ ! -f "${PATCH_DIR}/patch" - && -f "${patchfile}" ]]; then - cp "${patchfile}" "${PATCH_DIR}/patch" - if [[ $? == 0 ]] ; then - echo "Patch file ${patchfile} copied to ${PATCH_DIR}" - else - yetus_error "ERROR: Could not copy ${patchfile} to ${PATCH_DIR}" - cleanup_and_exit 1 - fi - fi -} - -## @description if patch-level zero, then verify we aren't -## @description just adding files -## @audience public -## @stability stable -## @param log filename -## @replaceable no -## @return $? -function patchfile_verify_zero -{ - declare logfile=$1 - shift - declare dir - declare changed_files1 - declare changed_files2 - declare filename - - # don't return /dev/null - # shellcheck disable=SC2016 - changed_files1=$(${AWK} 'function p(s){if(s!~"^/dev/null"){print s}} - /^diff --git / { p($3); p($4) } - /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u) - - # maybe we interpreted the patch wrong? check the log file - # shellcheck disable=SC2016 - changed_files2=$(${GREP} -E '^[cC]heck' "${logfile}" \ - | ${AWK} '{print $3}' \ - | ${SED} -e 's,\.\.\.$,,g') - - for filename in ${changed_files1} ${changed_files2}; do - - # leading prefix = bad - if [[ ${filename} =~ ^(a|b)/ ]]; then - return 1 - fi - - # touching an existing file is proof enough - # that pl=0 is good - if [[ -f ${filename} ]]; then - return 0 - fi - - dir=$(dirname "${filename}" 2>/dev/null) - if [[ -n ${dir} && -d ${dir} ]]; then - return 0 - fi - done - - # ¯\_(ã)_/¯ - no way for us to know, all new files with no prefix! - yetus_error "WARNING: Patch only adds files; using patch level ${PATCH_LEVEL}" - return 0 -} - -## @description git apply dryrun -## @replaceable no -## @audience private -## @stability evolving -## @param path to patch file to dryrun -function gitapply_dryrun -{ - declare patchfile=$1 - declare prefixsize=${2:-0} - - while [[ ${prefixsize} -lt 4 - && -z ${PATCH_METHOD} ]]; do - yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \ - "${GIT}" apply --binary -v --check "-p${prefixsize}" "${patchfile}" - if [[ $? == 0 ]]; then - PATCH_LEVEL=${prefixsize} - PATCH_METHOD=gitapply - break - fi - ((prefixsize=prefixsize+1)) - done - - if [[ ${prefixsize} -eq 0 ]]; then - patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log" - if [[ $? != 0 ]]; then - PATCH_METHOD="" - PATCH_LEVEL="" - gitapply_dryrun "${patchfile}" 1 - fi - fi -} - -## @description patch patch dryrun -## @replaceable no -## @audience private -## @stability evolving -## @param path to patch file to dryrun -function patchcmd_dryrun -{ - declare patchfile=$1 - declare prefixsize=${2:-0} - - while [[ ${prefixsize} -lt 4 - && -z ${PATCH_METHOD} ]]; do - # shellcheck disable=SC2153 - yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \ - "${PATCH}" "-p${prefixsize}" -E --dry-run < "${patchfile}" - if [[ $? == 0 ]]; then - PATCH_LEVEL=${prefixsize} - PATCH_METHOD=patchcmd - break - fi - ((prefixsize=prefixsize+1)) - done - - if [[ ${prefixsize} -eq 0 ]]; then - patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log" - if [[ $? != 0 ]]; then - PATCH_METHOD="" - PATCH_LEVEL="" - patchcmd_dryrun "${patchfile}" 1 - fi - fi -} - -## @description driver for dryrun methods -## @replaceable no -## @audience private -## @stability evolving -## @param path to patch file to dryrun -function patchfile_dryrun_driver -{ - declare patchfile=$1 - declare method - - #shellcheck disable=SC2153 - for method in "${PATCH_METHODS[@]}"; do - if declare -f ${method}_dryrun >/dev/null; then - "${method}_dryrun" "${patchfile}" - fi - if [[ -n ${PATCH_METHOD} ]]; then - break - fi - done - - if [[ -n ${PATCH_METHOD} ]]; then - return 0 - fi - return 1 -} - -## @description git patch apply -## @replaceable no -## @audience private -## @stability evolving -## @param path to patch file to apply -function gitapply_apply -{ - declare patchfile=$1 - declare extraopts - - if [[ "${COMMITMODE}" = true ]]; then - extraopts="--whitespace=fix" - fi - - echo "Applying the patch:" - yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-apply.log" \ - "${GIT}" apply --binary ${extraopts} -v --stat --apply "-p${PATCH_LEVEL}" "${patchfile}" - ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-apply.log" -} - -## @description patch patch apply -## @replaceable no -## @audience private -## @stability evolving -## @param path to patch file to apply -function patchcmd_apply -{ - declare patchfile=$1 - - echo "Applying the patch:" - yetus_run_and_redirect "${PATCH_DIR}/apply-patch-patch-apply.log" \ - "${PATCH}" "-p${PATCH_LEVEL}" -E < "${patchfile}" - cat "${PATCH_DIR}/apply-patch-patch-apply.log" -} - -## @description driver for patch apply methods -## @replaceable no -## @audience private -## @stability evolving -## @param path to patch file to apply -function patchfile_apply_driver -{ - declare patchfile=$1 - declare gpg=$2 - - if declare -f ${PATCH_METHOD}_apply >/dev/null; then - "${PATCH_METHOD}_apply" "${patchfile}" "${gpg}" - if [[ $? -gt 0 ]]; then - return 1 - fi - else - yetus_error "ERROR: Patching method ${PATCH_METHOD} does not have a way to apply patches!" - return 1 - fi - return 0 -}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/reaper.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/reaper.sh b/precommit/core.d/reaper.sh deleted file mode 100755 index ce48c25..0000000 --- a/precommit/core.d/reaper.sh +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -REAPER_MODE=off # off, report, kill -declare -i REAPER_TOTAL_COUNT=0 -REAPER_DOCKER_ONLY=true -REAPER_ZOMBIE_MODULES=() -REAPER_ZOMBIE_LOGS=() -declare -a REAPER_NAMES - - -## @description Add a regex to the reaper's checklist -## @description NOTE: Users WILL override anything added before -## @description argument parsing! -## @stability evolving -## @audience public -## @replaceable no -function reaper_add_name -{ - yetus_add_array_element REAPER_NAMES "$1" -} - -## @description Reaper-specific usage -## @stability stable -## @audience private -## @replaceable no -function reaper_usage -{ - yetus_add_option "--reapermode={off,report,kill}" "Set unit test reaper mode (default: '${REAPER_MODE}')" - yetus_add_option "--reaperdockeronly=<bool>" "Only run the reaper in --docker (default: ${REAPER_DOCKER_ONLY})" - yetus_add_option "--reapernames=<list>" "List of regexs to search (default build tool dependent)" -} - -## @description Reaper-specific argument parsing -## @stability stable -## @audience private -## @replaceable no -## @param arguments -function reaper_parse_args -{ - declare i - - for i in "$@"; do - case ${i} in - --reapermode=*) - REAPER_MODE=${i#*=} - ;; - --reaperdockeronly=*) - REAPER_DOCKER_ONLY=${i#*=} - ;; - --reapernames=*) - yetus_comma_to_array REAPER_NAMES "${i#*=}" - ;; - esac - done - - # Running the reaper outside of Dockermode is very dangerous - - if [[ "${REAPER_DOCKER_ONLY}" = "true" && ${DOCKERMODE} = "false" ]]; then - REAPER_MODE="off" - return - fi - - # make sure REAPER_MODE is something valid and turn us on - # as necessary - if [[ "${REAPER_MODE}" = "report" || "${REAPER_MODE}" = "kill" ]]; then - add_test_format reaper - yetus_add_entry EXEC_MODES Reaper - else - REAPER_MODE="off" - fi - -} - -## @description Initialize the reaper -## @stability stable -## @audience private -## @replaceable yes -## @param arguments -function reaper_initialize -{ - determine_user -} - - - -## @description Reaper coprocessor function that -## @description runs outside the law -## @stability evolving -## @audience private -## @replaceable yes -function reaper_coproc_func -{ - declare line - declare i - declare module - declare filefrag - declare cmd - declare args - declare pid - declare -a pidlist - declare -i count - - echo "Reaper watching for: ${REAPER_NAMES[*]}" >> "${PATCH_DIR}/reaper.txt" - - while true; do - read -r cmd - case ${cmd} in - reap) - - read -r module - read -r logfile - - while read -r line; do - ((count=count+1)) - for i in "${REAPER_NAMES[@]}"; do - echo "${line}" | ${GREP} -E "${i}" >> "${PATCH_DIR}/${logfile}" - done - done < <(ps -u "${USER_ID}" -o pid= -o args=) - - pidlist=() - count=0 - while read -r line; do - ((count=count+1)) - pid=$(echo "${line}" | cut -f1 -d' ') - args=$(echo "${line}" | cut -f2- -d' ') - if [[ "${REAPER_MODE}" = "kill" ]]; then - pidlist+=("${pid}") - echo "Killing ${pid} ${args}" >> "${PATCH_DIR}/reaper.txt" 2>&1 - fi - done < <(cat "${PATCH_DIR}/${logfile}") - - # tell our parent how many - # doing this now means killing in the background - echo ${count} - - if [[ ${count} -eq 0 ]]; then - rm "${PATCH_DIR}/${logfile}" - fi - - for i in "${pidlist[@]}"; do - if [[ "${REAPER_MODE}" = "kill" ]]; then - pid_kill "${i}" >> "${PATCH_DIR}/reaper.txt" 2>&1 - fi - done - ;; - exit) - exit 0 - ;; - esac - done -} - -## @description Run the reaper -## @stability evolving -## @audience private -## @replaceable yes -## @param module -## @param testlog -## @param testfrag -function reaper_post_exec -{ - declare module=$1 - declare filefrag=$2 - declare count - declare myfile="${filefrag}-reaper.txt" - declare killmsg="" - - case "${REAPER_MODE}" in - off) - return 0 - ;; - kill) - killmsg=" and killed" - ;; - esac - - yetus_debug "Checking for unreaped processes:" - - # give some time for things to die naturally - sleep 2 - - #shellcheck disable=SC2154,SC2086 - printf "reap\n%s\n%s\n" "${module}" "${myfile}" >&${reaper_coproc[1]} - - #shellcheck disable=SC2154,SC2086 - read -r count <&${reaper_coproc[0]} - - if [[ ${count} -gt 0 ]]; then - ((REAPER_TOTAL_COUNT=REAPER_TOTAL_COUNT+count)) - printf "\nFound%s %s left over processes\n\n" "${killmsg}" "${count}" - REAPER_ZOMBIE_MODULES+=("${module}:${count}") - REAPER_ZOMBIE_LOGS+=("@@BASE@@/${myfile}") - return 1 - fi - - return 0 -} - -## @description Reaper output to the user -## @stability evolving -## @audience private -## @replaceable yes -## @param jdkname -function reaper_finalize_results -{ - declare jdk=$1 - declare fn - - if [[ "${REAPER_MODE}" = "off" ]]; then - return 0 - fi - - if [[ ${#REAPER_ZOMBIE_MODULES[@]} -gt 0 ]] ; then - populate_test_table "${jdk}Unreaped Processes" "${REAPER_ZOMBIE_MODULES[@]}" - for fn in "${REAPER_ZOMBIE_LOGS[@]}"; do - add_footer_table "Unreaped Processes Log" "${fn}" - done - REAPER_ZOMBIE_MODULES=() - REAPER_ZOMBIE_LOGS=() - fi -} - -## @description Reaper output to the user -## @stability evolving -## @audience private -## @replaceable yes -## @param jdkname -function reaper_total_count -{ - - if [[ "${REAPER_MODE}" = "off" ]]; then - return 0 - fi - - if [[ ${REAPER_TOTAL_COUNT} -gt 0 ]]; then - add_vote_table -0 reaper "Unreaped process count: ${REAPER_TOTAL_COUNT}" - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/docker-cleanup.sh ---------------------------------------------------------------------- diff --git a/precommit/docker-cleanup.sh b/precommit/docker-cleanup.sh deleted file mode 100755 index 1d17b1b..0000000 --- a/precommit/docker-cleanup.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# no shelldocs required from this file -# SHELLDOC-IGNORE - -# Make sure that bash version meets the pre-requisite - -if [[ -z "${BASH_VERSINFO[0]}" ]] \ - || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ - || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then - echo "bash v3.2+ is required. Sorry." - exit 1 -fi - -this="${BASH_SOURCE-$0}" -BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -BINNAME=${this##*/} -BINNAME=${BINNAME%.sh} -#shellcheck disable=SC2034 -STARTINGDIR=$(pwd) -#shellcheck disable=SC2034 -USER_PARAMS=("$@") -#shellcheck disable=SC2034 -QATESTMODE=false - -# dummy functions -function add_vote_table -{ - true -} - -function add_footer_table -{ - true -} - -function bugsystem_finalreport -{ - true -} - -## @description import core library routines -## @audience private -## @stability evolving -function import_core -{ - declare filename - - for filename in "${BINDIR}/core.d"/*; do - # shellcheck disable=SC1091 - # shellcheck source=core.d/01-common.sh - . "${filename}" - done -} - -## @description import plugins then remove the stuff we don't need -## @audience public -## @stability stable -## @replaceable no -function import_and_clean -{ - importplugins - yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" - unset BUILDTOOLS - unset TESTTYPES - unset TESTFORMATS - - #shellcheck disable=SC2034 - DOCKER_CLEANUP_CMD=true - #shellcheck disable=SC2034 - DOCKERSUPPORT=true - #shellcheck disable=SC2034 - ROBOT=true - #shellcheck disable=SC2034 - DOCKERFAIL="fail" -} - -## @description Clean the filesystem as appropriate and then exit -## @audience private -## @stability evolving -## @replaceable no -## @param runresult -function cleanup_and_exit -{ - local result=$1 - - if [[ ${PATCH_DIR} =~ ^/tmp/yetus - && -d ${PATCH_DIR} ]]; then - rm -rf "${PATCH_DIR}" - fi - - # shellcheck disable=SC2086 - exit ${result} -} - -## @description Setup the default global variables -## @audience public -## @stability stable -## @replaceable no -function setup_defaults -{ - common_defaults -} - -## @description Interpret the command line parameters -## @audience private -## @stability stable -## @replaceable no -## @param $@ -## @return May exit on failure -function parse_args -{ - declare i - - common_args "$@" - - for i in "$@"; do - case ${i} in - --robot) - # shellcheck disable=SC2034 - ROBOT=true - ;; - --sentinel) - # shellcheck disable=SC2034 - ROBOT=true - # shellcheck disable=SC2034 - SENTINEL=true - yetus_add_entry EXEC_MODES Sentinel - ;; - esac - done - - docker_parse_args "$@" -} - -## @description Print the usage information -## @audience public -## @stability stable -## @replaceable no -function yetus_usage -{ - import_and_clean - - echo "${BINNAME} [OPTIONS]" - - yetus_add_option "--debug" "If set, then output some extra stuff to stderr" - yetus_add_option "--robot" "Assume this is an automated run" - yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)" - docker_usage - - yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" - yetus_reset_usage -} - -## @description Large display for the user console -## @audience public -## @stability stable -## @replaceable no -## @param string -## @return large chunk of text -function big_console_header -{ - local text="$*" - local spacing=$(( (75+${#text}) /2 )) - printf "\n\n" - echo "============================================================================" - echo "============================================================================" - printf "%*s\n" ${spacing} "${text}" - echo "============================================================================" - echo "============================================================================" - printf "\n\n" -} - -trap "cleanup_and_exit 1" HUP INT QUIT TERM - -import_core - -setup_defaults - -parse_args "$@" - -import_and_clean - -parse_args_plugins "$@" - -docker_initialize -plugins_initialize - -docker_cleanup -RESULT=$? - -cleanup_and_exit ${RESULT} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/it/it-tp.sh ---------------------------------------------------------------------- diff --git a/precommit/it/it-tp.sh b/precommit/it/it-tp.sh deleted file mode 100755 index 0e275f4..0000000 --- a/precommit/it/it-tp.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# where to get Apache Yetus -YETUSDIR=${YETUSDIR:-$(pwd)/yetus} - -# project to work with -PROJECT=${PROJECT:-hadoop} - -# temp directory to play in -WORKDIR=${WORKDIR:-/tmp/yetus.${PROJECT}} - -# where to store artifacts: logs, etc. RELATIVE PATH! -PATCHDIR=out - -# set the real dir where our source is. this is a relative path! -BASEDIR=srcdir - -# dockerfile to use -DOCKERFILE=${DOCKERFILE:-${BASEDIR}/dev-support/docker/Dockerfile} - -# make the directory if it doesn't exist -mkdir -p "${WORKDIR}" - -# build out workdir -pushd "${WORKDIR}" || exit 1 - -# checkout the project's source -if [[ ! -d "${BASEDIR}" ]]; then - git clone "https://github.com/apache/${PROJECT}"; "${BASEDIR}" -fi - -# clean out the out dir -rm -rf "${PATCHDIR}" || true - -# make sure they exist -mkdir -p "${PATCHDIR}" - -# if we abort the run in the middle of git, it will leave a present we -# don't want -if [[ -f "${BASEDIR}/.git/index.lock" ]]; then - rm "${BASEDIR}/.git/index.lock" -fi - -# our 'default' args, in (mostly) alphabetical order - -# rsync these files back into the archive dir -YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") - -# where the source is located -YETUS_ARGS+=("--basedir=${BASEDIR}") - -# want to make sure the output is sane for these -YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt") -YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt") -YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html") - -# run in docker mode -YETUS_ARGS+=("--docker") - -# which Dockerfile to use -YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") - -# force JDK to be OpenJDK 8 -YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") - -# temp storage, etc -YETUS_ARGS+=("--patch-dir=${PATCHDIR}") - -# plugins to enable. modify as necessary based upon what is being tested -YETUS_ARGS+=("--plugins=jira,maven,briefreport,htmlout") - -# Many projects need a high process limit -YETUS_ARGS+=("--proclimit=5000") - -# project name. this will auto trigger personality for built-ins -YETUS_ARGS+=("--project=${PROJECT}") - -# nuke the src repo before working -YETUS_ARGS+=("--resetrepo") - -# run test-patch from the source tree specified up above -TESTPATCHBIN=${YETUSDIR}/precommit/test-patch.sh - -# now run test-patch with any optional arguments: -# --empty-patch for a full run aka 'qbt' -# URL for a remote patch file -# file name for local patch file -# JIRA Issue, etc, etc. -# -# also, can add parameters or override the above as necessary - -cat <<EOF - -******************* -Starting test-patch -******************* - -EOF - -/bin/bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${@}" - -cat <<EOF - -******************* -Stopping test-patch -******************* - -EOF - -popd || exit 1 http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/jenkins/jenkins-admin.py ---------------------------------------------------------------------- diff --git a/precommit/jenkins/jenkins-admin.py b/precommit/jenkins/jenkins-admin.py deleted file mode 100755 index f6ffdfe..0000000 --- a/precommit/jenkins/jenkins-admin.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from optparse import OptionParser -from tempfile import NamedTemporaryFile -from xml.etree import ElementTree -import base64 -import httplib -import os -import re -import sys -import urllib2 - -def httpGet(resource, ignoreError=False, username=None, password=None): - request = urllib2.Request(resource) - if username and password: - base64string = base64.b64encode('%s:%s' % (username, password)) - request.add_header("Authorization", "Basic %s" % base64string) - try: - response = urllib2.urlopen(request) - except urllib2.HTTPError, http_err: - code = http_err.code - print '%s returns HTTP error %d: %s' \ - % (resource, code, http_err.reason) - if ignoreError: - return '' - else: - print 'Aborting.' - sys.exit(1) - except urllib2.URLError, url_err: - print 'Error contacting %s: %s' % (resource, url_err.reason) - if ignoreError: - return '' - else: - raise url_err - except httplib.BadStatusLine, err: - if ignoreError: - return '' - else: - raise err - return response.read() - - -# returns a map of (project, issue) => attachment id - -def parseJiraData(fileName): - tree = ElementTree.parse(fileName) - root = tree.getroot() - jiraPattern = re.compile('([A-Z]+)\-([0-9]+)') - result = {} - for item in root.findall('./channel/item'): - jirakey = item.find('key') - if jirakey is None: - continue - jiraissue = jirakey.text - matcher = jiraPattern.match(jiraissue) - if not matcher: - continue - jiraissue = (matcher.group(1), matcher.group(2)) - attachmentIds = [] - for jiraattachment in item.findall('./attachments/attachment'): - attachmentId = jiraattachment.get('id') - try: - attachmentIds.append(int(attachmentId)) - except ValueError: - pass - if attachmentIds: - attachmentIds.sort() - result[jiraissue] = attachmentIds[-1] - return result - - -if __name__ == '__main__': - parser = OptionParser(prog = 'jenkins-admin') - if os.getenv('JENKINS_URL'): - parser.set_defaults(jenkinsUrl=os.getenv('JENKINS_URL')) - if os.getenv('JOB_NAME'): - parser.set_defaults(jenkinsJobName=os.getenv('JOB_NAME')) - else: - parser.set_defaults(jenkinsJobName='PreCommit-Admin') - - parser.set_defaults(jenkinsJobTemplate='PreCommit-{project}') - parser.add_option('--initialize', action='store_true', - dest='jenkinsInit', - help='Start a new patch_tested.txt file') - parser.add_option('--jenkins-jobname', type='string', - dest='jenkinsJobName', - help='PreCommit-Admin JobName', metavar='JOB_NAME') - parser.add_option('--jenkins-project-template', type='string', - dest='jenkinsJobTemplate', - help='Template for project jobs', - metavar='TEMPLATE') - parser.add_option('--jenkins-token', type='string', - dest='jenkinsToken', help='Jenkins Token', - metavar='TOKEN') - parser.add_option('--jenkins-url', type='string', dest='jenkinsUrl' - , help='Jenkins base URL', metavar='URL') - parser.add_option( - '--jenkins-url-override', - type='string', - dest='jenkinsUrlOverrides', - action='append', - help='Project specific Jenkins base URL', - metavar='PROJECT=URL', - ) - parser.add_option('--jira-filter', type='string', dest='jiraFilter', - help='JIRA filter URL', metavar='URL') - parser.add_option('--jira-user', type='string', dest='jiraUser', - help='JIRA username') - parser.add_option('--jira-password', type='string', dest='jiraPassword', - help='JIRA password') - parser.add_option('--live', dest='live', action='store_true', - help='Submit Job to jenkins') - parser.add_option('--max-history', dest='history', type='int', - help='Maximum history to store', default=5000) - parser.add_option( - '-V', - '--version', - dest='release_version', - action='store_true', - default=False, - help="display version information for jenkins-admin and exit.") - - (options, args) = parser.parse_args() - - # Handle the version string right away and exit - if options.release_version: - with open( - os.path.join( - os.path.dirname(__file__), "../../VERSION"), 'r') as ver_file: - print ver_file.read() - sys.exit(0) - - tokenFrag = '' - if options.jenkinsToken: - tokenFrag = 'token=%s' % options.jenkinsToken - else: - tokenFrag = 'token={project}-token' - if not options.jiraFilter: - parser.error('ERROR: --jira-filter is a required argument.') - if not options.jenkinsUrl: - parser.error('ERROR: --jenkins-url or the JENKINS_URL environment variable is required.' - ) - if options.history < 0: - parser.error('ERROR: --max-history must be 0 or a positive integer.' - ) - jenkinsUrlOverrides = {} - if options.jenkinsUrlOverrides: - for override in options.jenkinsUrlOverrides: - if '=' not in override: - parser.error('Invalid Jenkins Url Override: ' - + override) - (project, url) = override.split('=', 1) - jenkinsUrlOverrides[project.upper()] = url - tempFile = NamedTemporaryFile(delete=False) - try: - jobLogHistory = None - if not options.jenkinsInit: - jobLogHistory = httpGet(options.jenkinsUrl - + '/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt' - % options.jenkinsJobName, True) - - # if we don't have a successful build available try the last build - - if not jobLogHistory: - jobLogHistory = httpGet(options.jenkinsUrl - + '/job/%s/lastCompletedBuild/artifact/patch_tested.txt' - % options.jenkinsJobName) - jobLogHistory = jobLogHistory.strip().split('\n') - if 'TESTED ISSUES' not in jobLogHistory[0]: - print 'Downloaded patch_tested.txt control file may be corrupted. Failing.' - sys.exit(1) - - # we are either going to write a new one or rewrite the old one - - jobLog = open('patch_tested.txt', 'w+') - - if jobLogHistory: - if len(jobLogHistory) > options.history: - jobLogHistory = [jobLogHistory[0]] \ - + jobLogHistory[len(jobLogHistory) - - options.history:] - for jobHistoryRecord in jobLogHistory: - jobLog.write(jobHistoryRecord + '\n') - else: - jobLog.write('TESTED ISSUES\n') - jobLog.flush() - rssData = httpGet(options.jiraFilter,False,options.jiraUser,options.jiraPassword) - tempFile.write(rssData) - tempFile.flush() - for (key, attachment) in parseJiraData(tempFile.name).items(): - (project, issue) = key - if jenkinsUrlOverrides.has_key(project): - url = jenkinsUrlOverrides[project] - else: - url = options.jenkinsUrl - - jenkinsUrlTemplate = url + '/job/' \ - + options.jenkinsJobTemplate \ - + '/buildWithParameters?' + tokenFrag \ - + '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}' - - urlArgs = { - 'project': project, - 'issue': issue, - 'attachment': attachment, - } - jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs) - - # submit job - - jobName = '%s-%s,%s' % (project, issue, attachment) - if not jobLogHistory or jobName not in jobLogHistory: - print jobName + ' has not been processed, submitting' - jobLog.write(jobName + '\n') - jobLog.flush() - if options.live: - httpGet(jenkinsUrl, True) - else: - print 'GET ' + jenkinsUrl - else: - print jobName + ' has been processed, ignoring' - jobLog.close() - finally: - if options.live: - os.remove(tempFile.name) - else: - print 'JIRA Data is located: ' + tempFile.name http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/jenkins/unit-test-filter-file.example ---------------------------------------------------------------------- diff --git a/precommit/jenkins/unit-test-filter-file.example b/precommit/jenkins/unit-test-filter-file.example deleted file mode 100644 index dc38859..0000000 --- a/precommit/jenkins/unit-test-filter-file.example +++ /dev/null @@ -1,2 +0,0 @@ -[package].[class].[method] -![package].[class].[method] http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/accumulo.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/accumulo.sh b/precommit/personality/accumulo.sh deleted file mode 100755 index f2d3e7c..0000000 --- a/precommit/personality/accumulo.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run all plugins -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - BUILDTOOL=maven - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^ACCUMULO-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/accumulo" - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="http://accumulo.apache.org/git.html#contributors"; - # We want to invoke the 'check' goal, not the default 'checkstyle'. We define - # our own checkstyle rules which isn't possible via 'checkstyle' configuration - #shellcheck disable=SC2034 - CHECKSTYLE_GOAL="check" -} - -## @description Queue up modules for this personality -## @audience private -## @stability evolving -## @param repostatus -## @param testtype -function personality_modules -{ - local repostatus=$1 - local testtype=$2 - - yetus_debug "Personality: ${repostatus} ${testtype}" - clear_personality_queue - - if [[ ${testtype} == 'unit' ]]; then - # Run all tests, not just the tests in the modules affected - yetus_debug "Overriding to run all unit tests" - - personality_enqueue_module . - return - fi - - # Make sure we re-add the changed modules if we didn't short-circuit out - for module in "${CHANGED_MODULES[@]}"; do - personality_enqueue_module "${module}" - done -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/bigtop.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/bigtop.sh b/precommit/personality/bigtop.sh deleted file mode 100755 index c374289..0000000 --- a/precommit/personality/bigtop.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all,-checkstyle,-findbugs" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - # shellcheck disable=SC2034 - JIRA_ISSUE_RE='^BIGTOP-[0-9]+$' - # shellcheck disable=SC2034 - BUILDTOOL=gradle - # shellcheck disable=SC2034 - GITHUB_REPO="apache/bigtop" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/drill.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/drill.sh b/precommit/personality/drill.sh deleted file mode 100755 index cc35d45..0000000 --- a/precommit/personality/drill.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - BUILDTOOL=maven - #shellcheck disable=SC2034 - GITHUB_REPO="apache/drill" - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="https://drill.apache.org/docs/apache-drill-contribution-guidelines/"; - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^DRILL-[0-9]+$' - #shellcheck disable=SC2034 - JIRA_STATUS_RE='Reviewable' -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/flink.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/flink.sh b/precommit/personality/flink.sh deleted file mode 100755 index a9c0fdf..0000000 --- a/precommit/personality/flink.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - BUILDTOOL=maven - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^FLINK-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/flink" -} - -add_test_type flinklib - -## @description flinklib file filter -## @audience private -## @stability evolving -## @param filename -function flinklib_filefilter -{ - local filename=$1 - - if [[ ${filename} =~ \.java$ - || ${filename} =~ \.scala$ - || ${filename} =~ pom.xml$ ]]; then - add_test flinklib - fi -} - -## @description count files in the lib dir -## @audience private -## @stability evolving -function flinklib_count -{ - find "${BASEDIR}" \ - | ${GREP} "/lib/" \ - | ${GREP} -v "_qa_workdir" \ - | wc -l -} - -## @description check fliblib -## @audience private -## @stability evolving -## @param repostatus -function flinklib_rebuild -{ - declare repostatus=$1 - - start_clock - big_console_header "${PATCH_BRANCH} flink library dependencies" - - if ! verify_needed_test flinklib; then - echo "${BUILDMODEMSG} does not need flinklib testing." - return 0 - fi - - pushd "${BASEDIR}" >/dev/null - echo_and_redirect "${PATCH_DIR}/${repostatus}-flinklib-root.txt" \ - "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch - if [[ $? != 0 ]]; then - add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}." - fi - - if [[ ${repostatus} = branch ]]; then - FLINK_PRE_LIB_FILES=$(flinklib_count) - return 0 - else - FLINK_POST_LIB_FILES=$(flinklib_count) - fi - popd >/dev/null - - if [[ "${BUILDMODE}" = full ]]; then - if [[ ${FLINK_POST_LIB_FILES} -gt 0 ]]; then - add_vote_table -1 flinklib "Lib folder dependencies are currently ${FLINK_POST_LIB_FILES}" - return 1 - else - add_vote_table +1 flinklib "No lib folder dependencies!" - return 0 - fi - fi - - if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then - add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \ - "${FLINK_PRE_LIB_FILES} to ${FLINK_POST_LIB_FILES}" - return 1 - elif [[ "${FLINK_POST_LIB_FILES}" -eq "${FLINK_PRE_LIB_FILES}" ]]; then - add_vote_table 0 flinklib "Patch did not change lib dependencies" \ - " (still ${FLINK_PRE_LIB_FILES})" - else - add_vote_table +1 flinklib "Patch decreases lib folder dependencies by " \ - "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))." - fi - return 0 -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/geode.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/geode.sh b/precommit/personality/geode.sh deleted file mode 100755 index 9f5c0ec..0000000 --- a/precommit/personality/geode.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all,-ant,-javac,-scalac,-scaladoc" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=develop - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/GEODE/How+to+Contribute"; - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^(GEODE)-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/incubator-geode" - #shellcheck disable=SC2034 - BUILDTOOL=gradle -# PYLINT_OPTIONS="--indent-string=' '" - -# HADOOP_MODULES="" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/hadoop.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh deleted file mode 100755 index 2ad4459..0000000 --- a/precommit/personality/hadoop.sh +++ /dev/null @@ -1,563 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# SHELLDOC-IGNORE -# -# Override these to match Apache Hadoop's requirements - -personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - BUILDTOOL=maven - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=trunk - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="https://wiki.apache.org/hadoop/HowToContribute"; - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS|HDDS)-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/hadoop" - #shellcheck disable=SC2034 - PYLINT_OPTIONS="--indent-string=' '" - - HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)} - if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then - HADOOP_HOMEBREW_DIR=/usr/local - fi -} - -function personality_parse_args -{ - declare i - - for i in "$@"; do - case ${i} in - --hadoop-isal-prefix=*) - ISAL_HOME=${i#*=} - ;; - --hadoop-openssl-prefix=*) - OPENSSL_HOME=${i#*=} - ;; - --hadoop-snappy-prefix=*) - SNAPPY_HOME=${i#*=} - ;; - esac - done -} - -## @description Calculate the actual module ordering -## @audience private -## @stability evolving -## @param ordering -function hadoop_order -{ - declare ordering=$1 - declare hadoopm - - if [[ ${ordering} = normal ]]; then - hadoopm="${CHANGED_MODULES[*]}" - elif [[ ${ordering} = union ]]; then - hadoopm="${CHANGED_UNION_MODULES}" - elif [[ ${ordering} = mvnsrc ]]; then - hadoopm="${MAVEN_SRC_MODULES[*]}" - elif [[ ${ordering} = mvnsrctest ]]; then - hadoopm="${MAVEN_SRCTEST_MODULES[*]}" - else - hadoopm="${ordering}" - fi - echo "${hadoopm}" -} - -## @description Determine if it is safe to run parallel tests -## @audience private -## @stability evolving -## @param ordering -function hadoop_test_parallel -{ - if [[ -f "${BASEDIR}/pom.xml" ]]; then - HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \ - | head -1 \ - | "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \ - | cut -f1 -d- ) - export HADOOP_VERSION - else - return 1 - fi - - hmajor=${HADOOP_VERSION%%\.*} - hmajorminor=${HADOOP_VERSION%\.*} - hminor=${hmajorminor##*\.} - # ... and just for reference - #hmicro=${HADOOP_VERSION##*\.} - - # Apache Hadoop v2.8.0 was the first one to really - # get working parallel unit tests - if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then - return 1 - fi - - return 0 -} - -## @description Install extra modules for unit tests -## @audience private -## @stability evolving -## @param ordering -function hadoop_unittest_prereqs -{ - declare input=$1 - declare mods - declare need_common=0 - declare building_common=0 - declare module - declare flags - declare fn - - # prior to running unit tests, hdfs needs libhadoop.so built - # if we're building root, then this extra work is moot - - #shellcheck disable=SC2086 - mods=$(hadoop_order ${input}) - - for module in ${mods}; do - if [[ ${module} = hadoop-hdfs-project* ]]; then - need_common=1 - elif [[ ${module} = hadoop-common-project/hadoop-common - || ${module} = hadoop-common-project ]]; then - building_common=1 - elif [[ ${module} = . ]]; then - return - fi - done - - # Windows builds *ALWAYS* need hadoop-common compiled - case ${OSTYPE} in - Windows_NT|CYGWIN*|MINGW*|MSYS*) - need_common=1 - ;; - esac - - if [[ ${need_common} -eq 1 - && ${building_common} -eq 0 ]]; then - echo "unit test pre-reqs:" - module="hadoop-common-project/hadoop-common" - fn=$(module_file_fragment "${module}") - flags="$(hadoop_native_flags) $(yarn_ui2_flag)" - pushd "${BASEDIR}/${module}" >/dev/null - # shellcheck disable=SC2086 - echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ - "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} - popd >/dev/null - fi -} - -## @description Calculate the flags/settings for yarn-ui v2 build -## @description based upon the OS -## @audience private -## @stability evolving -function yarn_ui2_flag -{ - - if [[ ${BUILD_NATIVE} != true ]]; then - return - fi - - # Now it only tested on Linux/OSX, don't enable the profile on - # windows until it get verified - case ${OSTYPE} in - Linux) - # shellcheck disable=SC2086 - echo -Pyarn-ui - ;; - Darwin) - echo -Pyarn-ui - ;; - *) - # Do nothing - ;; - esac -} - -## @description Calculate the flags/settings for native code -## @description based upon the OS -## @audience private -## @stability evolving -function hadoop_native_flags -{ - if [[ ${BUILD_NATIVE} != true ]]; then - return - fi - - declare -a args - - # Based upon HADOOP-11937 - # - # Some notes: - # - # - getting fuse to compile on anything but Linux - # is always tricky. - # - Darwin assumes homebrew is in use. - # - HADOOP-12027 required for bzip2 on OS X. - # - bzip2 is broken in lots of places - # (the shared library is considered experimental) - # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 - # - - args=("-Drequire.test.libhadoop") - - if [[ -d "${ISAL_HOME}/include" ]]; then - args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}") - fi - - if [[ -d "${OPENSSL_HOME}/include" ]]; then - args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}") - elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then - args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/") - fi - - if [[ -d "${SNAPPY_HOME}/include" ]]; then - args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}") - elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then - args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy") - fi - - case ${OSTYPE} in - Linux) - # shellcheck disable=SC2086 - echo \ - -Pnative \ - -Drequire.fuse \ - -Drequire.openssl \ - -Drequire.snappy \ - -Drequire.valgrind \ - "${args[@]}" - ;; - Darwin) - echo \ - "${args[@]}" \ - -Pnative \ - -Drequire.snappy \ - -Drequire.openssl - ;; - Windows_NT|CYGWIN*|MINGW*|MSYS*) - echo \ - "${args[@]}" \ - -Drequire.snappy -Drequire.openssl -Pnative-win - ;; - *) - echo \ - "${args[@]}" - ;; - esac -} - -## @description Queue up modules for this personality -## @audience private -## @stability evolving -## @param repostatus -## @param testtype -function personality_modules -{ - declare repostatus=$1 - declare testtype=$2 - declare extra="" - declare ordering="normal" - declare needflags=false - declare foundbats=false - declare flags - declare fn - declare i - declare hadoopm - - yetus_debug "Personality: ${repostatus} ${testtype}" - - clear_personality_queue - - case ${testtype} in - asflicense) - # this is very fast and provides the full path if we do it from - # the root of the source - personality_enqueue_module . - return - ;; - checkstyle) - ordering="union" - extra="-DskipTests" - ;; - compile) - ordering="union" - extra="-DskipTests" - needflags=true - - # if something in common changed, we build the whole world - if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then - yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " - ordering="." - fi - ;; - distclean) - ordering="." - extra="-DskipTests" - ;; - javadoc) - if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then - ordering=. - fi - - if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then - echo "javadoc pre-reqs:" - for i in hadoop-project \ - hadoop-common-project/hadoop-annotations; do - fn=$(module_file_fragment "${i}") - pushd "${BASEDIR}/${i}" >/dev/null - echo "cd ${i}" - echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \ - "${MAVEN}" "${MAVEN_ARGS[@]}" install - popd >/dev/null - done - fi - extra="-Pdocs -DskipTests" - ;; - mvneclipse) - if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then - ordering=. - fi - ;; - mvninstall) - extra="-DskipTests" - if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then - ordering=. - fi - ;; - mvnsite) - if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then - ordering=. - fi - ;; - unit) - if [[ "${BUILDMODE}" = full ]]; then - ordering=mvnsrc - elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then - ordering=. - fi - - if [[ ${TEST_PARALLEL} = "true" ]] ; then - if hadoop_test_parallel; then - extra="-Pparallel-tests" - if [[ -n ${TEST_THREADS:-} ]]; then - extra="${extra} -DtestsThreadCount=${TEST_THREADS}" - fi - fi - fi - needflags=true - hadoop_unittest_prereqs "${ordering}" - - if ! verify_needed_test javac; then - yetus_debug "hadoop: javac not requested" - if ! verify_needed_test native; then - yetus_debug "hadoop: native not requested" - yetus_debug "hadoop: adding -DskipTests to unit test" - extra="-DskipTests" - fi - fi - - for i in "${CHANGED_FILES[@]}"; do - if [[ "${i}" =~ \.bats ]]; then - foundbats=true - fi - done - - if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then - yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile" - extra="${extra} -P!shelltest" - else - extra="${extra} -Pshelltest" - fi - ;; - *) - extra="-DskipTests" - ;; - esac - - if [[ ${needflags} = true ]]; then - flags="$(hadoop_native_flags) $(yarn_ui2_flag)" - extra="${extra} ${flags}" - fi - - extra="-Ptest-patch ${extra}" - - for module in $(hadoop_order ${ordering}); do - # shellcheck disable=SC2086 - personality_enqueue_module ${module} ${extra} - done -} - -## @description Add tests based upon personality needs -## @audience private -## @stability evolving -## @param filename -function personality_file_tests -{ - declare filename=$1 - - yetus_debug "Using Hadoop-specific personality_file_tests" - - if [[ ${filename} =~ src/main/webapp ]]; then - yetus_debug "tests/webapp: ${filename}" - add_test shadedclient - elif [[ ${filename} =~ \.sh - || ${filename} =~ \.cmd - || ${filename} =~ src/scripts - || ${filename} =~ src/test/scripts - || ${filename} =~ src/main/bin - || ${filename} =~ shellprofile\.d - || ${filename} =~ src/main/conf - ]]; then - yetus_debug "tests/shell: ${filename}" - add_test mvnsite - add_test unit - elif [[ ${filename} =~ \.md$ - || ${filename} =~ \.md\.vm$ - || ${filename} =~ src/site - ]]; then - yetus_debug "tests/site: ${filename}" - add_test mvnsite - elif [[ ${filename} =~ \.c$ - || ${filename} =~ \.cc$ - || ${filename} =~ \.h$ - || ${filename} =~ \.hh$ - || ${filename} =~ \.proto$ - || ${filename} =~ \.cmake$ - || ${filename} =~ CMakeLists.txt - ]]; then - yetus_debug "tests/units: ${filename}" - add_test compile - add_test cc - add_test mvnsite - add_test javac - add_test unit - elif [[ ${filename} =~ build.xml$ - || ${filename} =~ pom.xml$ - || ${filename} =~ \.java$ - || ${filename} =~ src/main - ]]; then - yetus_debug "tests/javadoc+units: ${filename}" - add_test compile - add_test javac - add_test javadoc - add_test mvninstall - add_test mvnsite - add_test unit - add_test shadedclient - fi - - # if we change anything in here, e.g. the test scripts - # then run the client artifact tests - if [[ ${filename} =~ hadoop-client-modules ]]; then - add_test shadedclient - fi - - if [[ ${filename} =~ src/test ]]; then - yetus_debug "tests: src/test" - add_test unit - fi - - if [[ ${filename} =~ \.java$ ]]; then - add_test findbugs - fi -} - -## @description Image to print on success -## @audience private -## @stability evolving -function hadoop_console_success -{ - printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; - printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; - printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; - printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; - printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; - printf "ICB8bXwgICB8bXwgIAo" -} - -################################################### -# Hadoop project specific check of IT for shaded artifacts - -add_test_type shadedclient - -## @description check for test modules and add test/plugins as needed -## @audience private -## @stability evolving -function shadedclient_initialize -{ - maven_add_install shadedclient -} - -## @description build client facing shaded artifacts and test them -## @audience private -## @stability evolving -## @param repostatus -function shadedclient_rebuild -{ - declare repostatus=$1 - declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt" - declare module - declare -a modules=() - - if [[ ${OSTYPE} = Windows_NT || - ${OSTYPE} =~ ^CYGWIN.* || - ${OSTYPE} =~ ^MINGW32.* || - ${OSTYPE} =~ ^MSYS.* ]]; then - echo "hadoop personality: building on windows, skipping check of client artifacts." - return 0 - fi - - yetus_debug "hadoop personality: seeing if we need the test of client artifacts." - for module in hadoop-client-modules/hadoop-client-check-invariants \ - hadoop-client-modules/hadoop-client-check-test-invariants \ - hadoop-client-modules/hadoop-client-integration-tests; do - if [ -d "${module}" ]; then - yetus_debug "hadoop personality: test module '${module}' is present." - modules=( ${modules[@]} -pl ${module} ) - fi - done - if [ ${#modules[@]} -eq 0 ]; then - echo "hadoop personality: no test modules present, skipping check of client artifacts." - return 0 - fi - - big_console_header "Checking client artifacts on ${repostatus}" - - echo_and_redirect "${logfile}" \ - "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ - "${modules[@]}" \ - -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true - - count=$(${GREP} -c '\[ERROR\]' "${logfile}") - if [[ ${count} -gt 0 ]]; then - add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts." - return 1 - fi - - add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts." - return 0 -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/hbase.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/hbase.sh b/precommit/personality/hbase.sh deleted file mode 100755 index b48691e..0000000 --- a/precommit/personality/hbase.sh +++ /dev/null @@ -1,368 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - BUILDTOOL=maven - #shellcheck disable=SC2034 - PROJECT_NAME=hbase - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^HBASE-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/hbase" - - # TODO use PATCH_BRANCH to select hadoop versions to use. - # All supported Hadoop versions that we want to test the compilation with - HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1" - - # TODO use PATCH_BRANCH to select jdk versions to use. - - # Override the maven options - MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}" - -} - -## @description Queue up modules for this personality -## @audience private -## @stability evolving -## @param repostatus -## @param testtype -function personality_modules -{ - local repostatus=$1 - local testtype=$2 - local extra="" - - yetus_debug "Personality: ${repostatus} ${testtype}" - - clear_personality_queue - - extra="-DHBasePatchProcess" - - if [[ ${repostatus} == branch - && ${testtype} == mvninstall ]] || - [[ "${BUILDMODE}" == full ]];then - personality_enqueue_module . ${extra} - return - fi - - if [[ ${testtype} = findbugs ]]; then - for module in "${CHANGED_MODULES[@]}"; do - # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing - # in src/main/java where findbugs goes to look - if [[ ${module} == hbase-shell ]]; then - continue - elif [[ ${module} == hbase-it ]]; then - continue - else - # shellcheck disable=SC2086 - personality_enqueue_module ${module} ${extra} - fi - done - return - fi - - if [[ ${testtype} = unit ]]; then - extra="${extra} -PrunAllTests" - - # Inject the jenkins build-id for our surefire invocations - # Used by zombie detection stuff, even though we're not including that yet. - if [ -n "${BUILD_ID}" ]; then - extra="${extra} -Dbuild.id=${BUILD_ID}" - fi - fi - - for module in "${CHANGED_MODULES[@]}"; do - # shellcheck disable=SC2086 - personality_enqueue_module ${module} ${extra} - done -} - -################################################### -# Below here are our one-off tests specific to hbase. -# TODO break them into individual files so it's easier to maintain them? - -# TODO line length check? could ignore all java files since checkstyle gets them. - -################################################### - -add_test_type hadoopcheck - -## @description hadoopcheck file filter -## @audience private -## @stability evolving -## @param filename -function hadoopcheck_filefilter -{ - local filename=$1 - - if [[ ${filename} =~ \.java$ ]]; then - add_test hadoopcheck - fi -} - -## @description hadoopcheck test -## @audience private -## @stability evolving -## @param repostatus -function hadoopcheck_rebuild -{ - local repostatus=$1 - local hadoopver - local logfile - local count - local result=0 - - if [[ "${repostatus}" = branch ]]; then - return 0 - fi - - big_console_header "Compiling against various Hadoop versions" - - export MAVEN_OPTS="${MAVEN_OPTS}" - for hadoopver in ${HBASE_HADOOP_VERSIONS}; do - logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt" - echo_and_redirect "${logfile}" \ - "${MAVEN}" clean install \ - -DskipTests -DHBasePatchProcess \ - -Dhadoop-two.version="${hadoopver}" - count=$(${GREP} -c ERROR "${logfile}") - if [[ ${count} -gt 0 ]]; then - add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}." - ((result=result+1)) - fi - done - - if [[ ${result} -gt 0 ]]; then - return 1 - fi - - add_vote_table +1 hadoopcheck "${BUILDMODEMSG} does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}." - return 0 -} - -###################################### - -# TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol -add_test_type hbaseprotoc - -## @description hbaseprotoc file filter -## @audience private -## @stability evolving -## @param filename -function hbaseprotoc_filefilter -{ - local filename=$1 - - if [[ ${filename} =~ \.proto$ ]]; then - add_test hbaseprotoc - fi -} - -## @description hadoopcheck test -## @audience private -## @stability evolving -## @param repostatus -function hbaseprotoc_rebuild -{ - declare repostatus=$1 - declare i=0 - declare fn - declare module - declare logfile - declare count - declare result - - if [[ "${repostatus}" = branch ]]; then - return 0 - fi - - if ! verify_needed_test hbaseprotoc; then - return 0 - fi - - big_console_header "HBase protoc plugin: ${BUILDMODE}" - - start_clock - - personality_modules patch hbaseprotoc - modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess - - # shellcheck disable=SC2153 - until [[ $i -eq "${#MODULE[@]}" ]]; do - if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then - ((result=result+1)) - ((i=i+1)) - continue - fi - module=${MODULE[$i]} - fn=$(module_file_fragment "${module}") - logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt" - - count=$(${GREP} -c ERROR "${logfile}") - - if [[ ${count} -gt 0 ]]; then - module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\ - "${count} new protoc errors in ${module}." - ((result=result+1)) - fi - ((i=i+1)) - done - - modules_messages patch hbaseprotoc true - if [[ ${result} -gt 0 ]]; then - return 1 - fi - return 0 -} - -###################################### - -add_test_type hbaseanti - -## @description hbaseanti file filter -## @audience private -## @stability evolving -## @param filename -function hbaseanti_filefilter -{ - local filename=$1 - - if [[ ${filename} =~ \.java$ ]]; then - add_test hbaseanti - fi -} - -## @description hbaseanti patch file check -## @audience private -## @stability evolving -## @param filename -function hbaseanti_patchfile -{ - local patchfile=$1 - local warnings - local result - - if [[ "${BUILDMODE}" = full ]]; then - return 0 - fi - - if ! verify_needed_test hbaseanti; then - return 0 - fi - - big_console_header "Checking for known anti-patterns" - - start_clock - - warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}") - if [[ ${warnings} -gt 0 ]]; then - add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}." - ((result=result+1)) - fi - - warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}") - if [[ ${warnings} -gt 0 ]]; then - add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}." - ((result=result+1)) - fi - - if [[ ${result} -gt 0 ]]; then - return 1 - fi - - add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns." - return 0 -} - - -## @description hbase custom mvnsite file filter. See HBASE-15042 -## @audience private -## @stability evolving -## @param filename -function mvnsite_filefilter -{ - local filename=$1 - - if [[ ${BUILDTOOL} = maven ]]; then - if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then - yetus_debug "tests/mvnsite: ${filename}" - add_test mvnsite - fi - fi -} - -## This is named so that yetus will check us right after running tests. -## Essentially, we check for normal failures and then we look for zombies. -#function hbase_unit_logfilter -#{ -# declare testtype="unit" -# declare input=$1 -# declare output=$2 -# declare processes -# declare process_output -# declare zombies -# declare zombie_count=0 -# declare zombie_process -# -# yetus_debug "in hbase-specific unit logfilter." -# -# # pass-through to whatever is counting actual failures -# if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then -# "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}" -# elif declare -f ${testtype}_logfilter >/dev/null; then -# "${testtype}_logfilter" "${input}" "${output}" -# fi -# -# start_clock -# if [ -n "${BUILD_ID}" ]; then -# yetus_debug "Checking for zombie test processes." -# processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}") -# if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then -# yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop." -# yetus_debug "${processes}" -# sleep 30 -# #shellcheck disable=SC2016 -# for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do -# # Test our zombie still running (and that it still an hbase build item) -# process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}") -# if [[ -n "${process_output}" ]]; then -# yetus_error "Zombie: ${process_output}" -# ((zombie_count = zombie_count + 1)) -# zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3) -# zombies="${zombies} ${zombie_process}" -# fi -# done -# fi -# if [ "${zombie_count}" -ne 0 ]; then -# add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)" -# populate_test_table "zombie unit tests" "${zombies}" -# else -# yetus_info "Zombie check complete. All test runs exited normally." -# stop_clock -# fi -# else -# add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies." -# fi -# -#} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/jmeter.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/jmeter.sh b/precommit/personality/jmeter.sh deleted file mode 100755 index b4e61b5..0000000 --- a/precommit/personality/jmeter.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all,-javadoc,-findbugs,-asflicense" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=trunk - # shellcheck disable=SC2034 - BUILDTOOL=ant - # shellcheck disable=SC2034 - GITHUB_REPO="apache/jmeter" - # shellcheck disable=SC2034 - JMETER_DOWNLOAD_JARS=false -} - -add_test_type jmeter - -## @description Personality usage options -## @audience private -## @stability evolving -function jmeter_usage -{ - yetus_add_option "--jmeter-download-jars=<bool>" "download third-party jars needed by ant build" -} - -## @description Process personality options -## @audience private -## @stability evolving -## @param arguments -function jmeter_parse_args -{ - declare i - - for i in "$@"; do - case ${i} in - --jmeter-download-jars=*) - JMETER_DOWNLOAD_JARS=${i#*=} - ;; - esac - done -} - -## @description Download jmetere dependencies -## @audience private -## @stability evolving -function jmeter_precheck -{ - if [[ ${JMETER_DOWNLOAD_JARS} = true ]]; then - pushd "${BASEDIR}" >/dev/null - echo_and_redirect "${PATCH_DIR}/jmeter-branch-download-jars.txt" "${ANT}" download_jars - popd >/dev/null - fi -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/kafka.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/kafka.sh b/precommit/personality/kafka.sh deleted file mode 100755 index 0c2f749..0000000 --- a/precommit/personality/kafka.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all,-checkstyle,-asflicense" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=trunk - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^KAFKA-[0-9]+$' - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="http://kafka.apache.org/contributing.html"; - # shellcheck disable=SC2034 - BUILDTOOL=gradle - #shellcheck disable=SC2034 - GITHUB_REPO="apache/kafka" -} - -## @description Queue up modules for this personality -## @audience private -## @stability evolving -## @param repostatus -## @param testtype -function personality_modules -{ - declare repostatus=$1 - declare testtype=$2 - declare module - declare extra="" - - yetus_debug "Using kafka personality_modules" - yetus_debug "Personality: ${repostatus} ${testtype}" - - clear_personality_queue - - case ${testtype} in - gradleboot) - # kafka's bootstrap is broken - if [[ ${testtype} == gradleboot ]]; then - pushd "${BASEDIR}" >/dev/null - echo_and_redirect "${PATCH_DIR}/kafka-configure-gradle.txt" gradle - popd >/dev/null - fi - ;; - compile) - extra="clean jar" - ;; - esac - - for module in "${CHANGED_MODULES[@]}"; do - # shellcheck disable=SC2086 - personality_enqueue_module "${module}" ${extra} - done -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/orc.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/orc.sh b/precommit/personality/orc.sh deleted file mode 100755 index 0582b17..0000000 --- a/precommit/personality/orc.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all,-checkstyle,-findbugs" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^ORC-[0-9]+$' - # shellcheck disable=SC2034 - BUILDTOOL=cmake - #shellcheck disable=SC2034 - GITHUB_REPO="apache/orc" - - # protobuf fails - #shellcheck disable=SC2034 - CMAKE_ROOT_BUILD=true -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/pig.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/pig.sh b/precommit/personality/pig.sh deleted file mode 100755 index 2014fda..0000000 --- a/precommit/personality/pig.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=trunk - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^PIG-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/pig" - #shellcheck disable=SC2034 - BUILDTOOL=ant -} - -## @description Queue up modules for this personality -## @audience private -## @stability evolving -## @param repostatus -## @param testtype -function personality_modules -{ - local repostatus=$1 - local testtype=$2 - local extra="" - - yetus_debug "Personality: ${repostatus} ${testtype}" - - clear_personality_queue - - extra="-DPigPatchProcess= " - - case ${testtype} in - findbugs) - # shellcheck disable=SC2034 - ANT_FINDBUGSXML="${BASEDIR}/build/test/findbugs/pig-findbugs-report.xml" - extra="-Dfindbugs.home=${FINDBUGS_HOME}" - ;; - compile) - extra="${extra} -Djavac.args=-Xlint -Dcompile.c++=yes clean piggybank" - ;; - javadoc) - extra="${extra} -Dforrest.home=${FORREST_HOME}" - ;; - unit) - extra="${extra} -Dtest.junit.output.format=xml -Dcompile.c++=yes -Dtest.output=yes test-core" - ;; - esac - - # shellcheck disable=SC2086 - personality_enqueue_module . ${extra} -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/ratis.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/ratis.sh b/precommit/personality/ratis.sh deleted file mode 100755 index d700512..0000000 --- a/precommit/personality/ratis.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - # shellcheck disable=SC2034 - BUILDTOOL=maven - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^RATIS-[0-9]+$' - #shellcheck disable=SC2034 - GITHUB_REPO="apache/incubator-ratis" -} http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/samza.sh ---------------------------------------------------------------------- diff --git a/precommit/personality/samza.sh b/precommit/personality/samza.sh deleted file mode 100755 index 430ffbb..0000000 --- a/precommit/personality/samza.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -personality_plugins "all,-checkstyle" - -## @description Globals specific to this personality -## @audience private -## @stability evolving -function personality_globals -{ - #shellcheck disable=SC2034 - PATCH_BRANCH_DEFAULT=master - #shellcheck disable=SC2034 - JIRA_ISSUE_RE='^SAMZA-[0-9]+$' - #shellcheck disable=SC2034 - PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/SAMZA/Contributor's+Corner" - # shellcheck disable=SC2034 - BUILDTOOL=gradle - #shellcheck disable=SC2034 - GITHUB_REPO="apache/samza" -}
