Repository: yetus Updated Branches: refs/heads/master 6566f91db -> 23c00565e
YETUS-70. add support for make/cmake/autoconf/ctest Signed-off-by: Sean Busbey <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/23c00565 Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/23c00565 Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/23c00565 Branch: refs/heads/master Commit: 23c00565ef7bab7506b06e9cf3bc45cf9eaaf9e5 Parents: 6566f91 Author: Allen Wittenauer <[email protected]> Authored: Sat Oct 10 09:04:14 2015 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Mon Jan 25 15:50:27 2016 -0800 ---------------------------------------------------------------------- .../in-progress/precommit-buildtools.md | 34 +++- precommit/test-patch-docker/Dockerfile | 12 +- precommit/test-patch.d/autoconf.sh | 204 +++++++++++++++++++ precommit/test-patch.d/checkstyle.sh | 53 ++--- precommit/test-patch.d/cmake.sh | 183 +++++++++++++++++ precommit/test-patch.d/ctest.sh | 89 ++++++++ precommit/test-patch.d/findbugs.sh | 8 +- precommit/test-patch.d/gradle.sh | 2 +- precommit/test-patch.d/make.sh | 164 +++++++++++++++ precommit/test-patch.sh | 199 +++++++++++++----- 10 files changed, 844 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/asf-site-src/source/documentation/in-progress/precommit-buildtools.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/in-progress/precommit-buildtools.md b/asf-site-src/source/documentation/in-progress/precommit-buildtools.md index 7c90e6c..1b31940 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-buildtools.md +++ b/asf-site-src/source/documentation/in-progress/precommit-buildtools.md @@ -30,7 +30,21 @@ add_build_tool <pluginname> * BUILDTOOLCWD - - If the build tool does not always run from the ${BASEDIR} and instead needs to change the current working directory to work on a specific module, then set this to true. The default is false. + - This variable determines where the build tool's command (as returned by pluginname\_executor) should actually execute. It should be one of three values: + + * basedir - always execute at the root of the source tree + * module - switch to the directory as given by the module being processed + * /(path) - change to the directory as given by this absolute path. If the path does not exist, it will be created. + + If /(path) is used, two special substitutions may be made: + + * @@@BASEDIR@@@ will be replaced with the root of the source tree + * @@@MODULEDIR@@@ will be replaced with the module name + + This allows for custom directories to be created and used as necessary. + + + The default is module. * UNSUPPORTED\_TEST @@ -76,7 +90,7 @@ For example, the gradle build tool does not have a standard way to execute check * pluginname\_reorder\_modules - - This functions allows the plugin to (re-)order the modules (e.g. based on the output of the maven dependency plugin). When called CHANGED_MODULES already contains all changed modules. It must be altered to have an effect. + - This functions allows the plugin to (re-)order the modules (e.g. based on the output of the maven dependency plugin). When called CHANGED\_MODULES already contains all changed modules. It must be altered to have an effect. * pluginname\_(test)\_logfilter @@ -102,12 +116,28 @@ test-patch always passes -noinput to Ant. This forces ant to be non-interactive In Docker mode, the `${HOME}/.ivy2` directory is shared amongst all invocations. +# autoconf Specific + +autoconf requires make to be enabled. autoreconf is always used to rebuild the configure scripte. + +## Command Arguments + +autoconf will always run configure with prefix set to a directory in the patch processing directory. To configure other flags, set the AUTCONF_CONF_FLAGS environment variable. + +# CMAKE Specific + +By default, cmake will create a 'build' directory and perform all work there. This may be changed either on the command line or via a personality setting. cmake requires make to be enabled. + # Gradle Specific The gradle plug-in always rebuilds the gradlew file and uses gradlew as the method to execute commands. In Docker mode, the `${HOME}/.gradle` directory is shared amongst all invocations. +# Make Specific + +No notes. + # Maven Specific ## Command Arguments http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch-docker/Dockerfile ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile index ee92c33..4e9fa5d 100644 --- a/precommit/test-patch-docker/Dockerfile +++ b/precommit/test-patch-docker/Dockerfile @@ -24,17 +24,11 @@ WORKDIR /root ###### RUN apt-get update && apt-get install --no-install-recommends -y \ git curl ant make maven \ - cmake gcc g++ protobuf-compiler \ + cmake gcc g++ pkg-config \ build-essential libtool \ - zlib1g-dev pkg-config libssl-dev \ - snappy libsnappy-dev \ - bzip2 libbz2-dev \ - libjansson-dev \ - fuse libfuse-dev \ - libcurl4-openssl-dev \ + autoconf automake \ python python2.7 pylint \ - ruby \ - openjdk-7-jdk \ + ruby openjdk-7-jdk \ libperl-critic-perl # Fixing the Apache commons / Maven dependency problem under Ubuntu: http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.d/autoconf.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/autoconf.sh b/precommit/test-patch.d/autoconf.sh new file mode 100755 index 0000000..2937e72 --- /dev/null +++ b/precommit/test-patch.d/autoconf.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRCMAKEIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool autoconf + +## @description autoconf usage hook +## @audience private +## @stability evolving +## @replaceable no +function autoconf_usage +{ + yetus_add_option "--autoconf-configure-flags=<cmd>" "Extra, non-'--prefix' 'configure' flags to use" +} + +## @description autoconf argument parser +## @audience private +## @stability evolving +## @param args +function autoconf_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --autoconf-configure-flags=*) + AUTOCONF_CONF_FLAGS=${i#*=} + ;; + esac + done +} + +## @description initialize autoconf +## @audience private +## @stability evolving +## @replaceable no +function autoconf_initialize +{ + if ! declare -f make_executor > /dev/null; then + yetus_error "ERROR: autoconf requires make to be enabled." + return 1 + fi +} + +## @description get the name of the autoconf build filename +## @audience private +## @stability evolving +## @replaceable no +## @return autoconf build file +function autoconf_buildfile +{ + echo "Makefile.am" +} + +## @description get the name of the autoconf binary +## @audience private +## @stability evolving +## @replaceable no +## @return filename +## @param params +function autoconf_executor +{ + make_executor "$@" +} + +## @description precompile for autoconf +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +function autoconf_precompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != autoconf ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + big_console_header "Pre-patch ${PATCH_BRANCH} autoconf verification" + else + big_console_header "Patch autoconf verification" + fi + + personality_modules "${repostatus}" autoreconf + + pushd "${BASEDIR}" >/dev/null + echo_and_redirect "${PATCH_DIR}/${repostatus}-autoconf-autoreconf" autoreconf -fi + result=$? + popd >/dev/null + + if [[ ${result} != 0 ]]; then + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table -1 autoreconf "${PATCH_BRANCH} unable to autoreconf" + else + add_vote_table -1 autoreconf "${repostatus} unable to autoreconf" + fi + add_footer_table "autoreconf" "@@BASE@@/${repostatus}-autoconf-autoreconf" + return 1 + else + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table +1 autoreconf "${PATCH_BRANCH} autoreconf successful" + else + add_vote_table +1 autoreconf "${repostatus} autoreconf successful" + fi + fi + + personality_modules "${repostatus}" configure + + pushd "${BASEDIR}" >/dev/null + #shellcheck disable=SC2086 + echo_and_redirect \ + "${PATCH_DIR}/${repostatus}-autoconf-configure" \ + ./configure \ + --prefix="${PATCH_DIR}/${repostatus}-install-dir" \ + ${AUTOCONF_CONF_FLAGS} + result=$? + popd >/dev/null + + if [[ ${result} != 0 ]]; then + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table -1 configure "${PATCH_BRANCH} unable to configure" + else + add_vote_table -1 configure "${repostatus} unable to configure" + fi + add_footer_table "configure" "@@BASE@@/${repostatus}-autoconf-configure" + return 1 + else + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + add_vote_table +1 configure "${PATCH_BRANCH} configure successful" + else + add_vote_table +1 configure "${repostatus} configure successful" + fi + fi + return 0 +} + +## @description autoconf worker +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +## @param test +function autoconf_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + if [[ "${tst}" = distclean ]]; then + modules_workers "${repostatus}" distclean distclean + else + make_modules_worker "$@" + fi +} + +## @description autoconf module queuer +## @audience private +## @stability evolving +## @replaceable no +function autoconf_builtin_personality_modules +{ + make_builtin_personality_modules "$@" +} + +## @description autoconf test determiner +## @audience private +## @stability evolving +## @replaceable no +## @param filename +function autoconf_builtin_personality_file_tests +{ + declare filename=$1 + + if [[ ${filename} =~ \.m4$ + || ${filename} =~ \.in$ ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test unit + else + make_builtin_personality_file_tests "${filename}" + fi +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.d/checkstyle.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/checkstyle.sh b/precommit/test-patch.d/checkstyle.sh index c645757..ce50b49 100755 --- a/precommit/test-patch.d/checkstyle.sh +++ b/precommit/test-patch.d/checkstyle.sh @@ -110,23 +110,22 @@ function checkstyle_calcdiffs ## @replaceable no function checkstyle_runner { - local repostatus=$1 - local tmp=${PATCH_DIR}/$$.${RANDOM} - local j - local i=0 - local fn - local savestart=${TIMER} - local savestop - local output - local logfile - local repo - local modulesuffix - local cmd - local logline - local text - local linenum - local codeline - + declare repostatus=$1 + declare tmp=${PATCH_DIR}/$$.${RANDOM} + declare j + declare i=0 + declare fn + declare savestart=${TIMER} + declare savestop + declare output + declare logfile + declare repo + declare modulesuffix + declare cmd + declare logline + declare text + declare linenum + declare codeline # first, let's clear out any previous run information modules_reset @@ -148,10 +147,7 @@ function checkstyle_runner output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt" logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt" - # set up the command line, etc, to build... - if [[ ${BUILDTOOLCWD} == true ]]; then - pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null - fi + buildtool_cwd case ${BUILDTOOL} in ant) @@ -177,13 +173,10 @@ function checkstyle_runner # almost certainly checkstyle output #shellcheck disable=SC2086 - echo ${cmd} "> ${logfile}" - #shellcheck disable=SC2086 - ${cmd} 2>&1 \ - | tee "${logfile}" \ - | ${GREP} ^/ \ - | ${SED} -e "s,${BASEDIR},.,g" \ - > "${tmp}" + echo_and_redirect "${logfile}" ${cmd} + ${GREP} ^/ "${logfile}" \ + | ${SED} -e "s,${BASEDIR},.,g" \ + > "${tmp}" if [[ $? == 0 ]] ; then module_status ${i} +1 "${logfile}" "${modulesuffix} in ${repo} passed checkstyle" @@ -243,9 +236,7 @@ function checkstyle_runner #shellcheck disable=SC2034 MODULE_STATUS_TIMER[${i}]=${savestop} - if [[ ${BUILDTOOLCWD} == true ]]; then - popd >/dev/null - fi + popd >/dev/null ((i=i+1)) done http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.d/cmake.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/cmake.sh b/precommit/test-patch.d/cmake.sh new file mode 100755 index 0000000..9608a1d --- /dev/null +++ b/precommit/test-patch.d/cmake.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRCMAKEIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool cmake + +CMAKE=cmake +CMAKE_BUILD_DIR="build" +CMAKE_ROOT_BUILD=true + +## @description cmake usage hook +## @audience private +## @stability evolving +## @replaceable no +function cmake_usage +{ + yetus_add_option "--cmake-build-dir=<cmd>" "build directory off of each module to use (default: ${CMAKE_BUILD_DIR})" + yetus_add_option "--cmake-cmd=<cmd>" "The 'cmake' command to use (default 'cmake')" + yetus_add_option "--cmake-root-build=<bool>" "Only build off of root, don't use modules (default: ${CMAKE_ROOT_BUILD})" +} + +## @description cmake argument parser +## @audience private +## @stability evolving +## @param args +function cmake_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --cmake-build-dir=*) + CMAKE_BUILD_DIR=${i#*=} + ;; + --cmake-cmd=*) + CMAKE=${i#*=} + ;; + --cmake-root-build=*) + CMAKE_ROOT_BUILD=${i#*=} + ;; + esac + done +} + +## @description initialize cmake +## @audience private +## @stability evolving +## @replaceable no +function cmake_initialize +{ + if ! declare -f make_executor > /dev/null; then + yetus_error "ERROR: cmake requires make to be enabled." + return 1 + fi +} + +## @description get the name of the cmake build filename +## @audience private +## @stability evolving +## @replaceable no +## @return cmake build file +function cmake_buildfile +{ + echo "CMakeLists.txt" +} + +## @description get the name of the cmake binary +## @audience private +## @stability evolving +## @replaceable no +## @return filename +## @param cmake params +function cmake_executor +{ + if [[ $1 = "CMakeLists.txt" ]]; then + echo "${CMAKE}" + else + make_executor "$@" + fi +} + +## @description precompile for cmake +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +function cmake_precompile +{ + declare repostatus=$1 + declare result=0 + + if [[ ${BUILDTOOL} != cmake ]]; then + return 0 + fi + + if [[ "${repostatus}" = branch ]]; then + # shellcheck disable=SC2153 + big_console_header "${PATCH_BRANCH} cmake CMakeLists.txt" + else + big_console_header "Patch cmake CMakeLists.txt" + fi + + personality_modules "${repostatus}" CMakeLists.txt + + modules_workers "${repostatus}" CMakeLists.txt @@@MODULEDIR@@@ + result=$? + modules_messages "${repostatus}" CMakeLists.txt true + if [[ ${result} != 0 ]]; then + return 1 + fi + return 0 +} + +## @description cmake worker +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +## @param test +function cmake_modules_worker +{ + make_modules_worker "$@" +} + +## @description cmake module manipulator +## @audience private +## @stability evolving +## @replaceable no +function cmake_reorder_modules +{ + if [[ "${CMAKE_ROOT_BUILD}" = true ]]; then + #shellcheck disable=SC2034 + BUILDTOOLCWD="@@@BASEDIR@@@/${CMAKE_BUILD_DIR}" + #shellcheck disable=SC2034 + CHANGED_MODULES="." + #shellcheck disable=SC2034 + CHANGED_UNION_MODULES="." + else + #shellcheck disable=SC2034 + BUILDTOOLCWD="@@@MODULEDIR@@@/${CMAKE_BUILD_DIR}" + fi +} + +## @description cmake module queuer +## @audience private +## @stability evolving +## @replaceable no +function cmake_builtin_personality_modules +{ + make_builtin_personality_modules "$@" +} + +## @description cmake test determiner +## @audience private +## @stability evolving +## @replaceable no +## @param filename +function cmake_builtin_personality_file_tests +{ + declare filename=$1 + + if [[ ${filename} =~ CMakeLists.txt$ ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test unit + else + make_builtin_personality_file_tests "${filename}" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.d/ctest.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/ctest.sh b/precommit/test-patch.d/ctest.sh new file mode 100755 index 0000000..94f2894 --- /dev/null +++ b/precommit/test-patch.d/ctest.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_test_format ctest + +CTEST_FAILED_TESTS="" + +## @description initialize ctest +## @audience private +## @stability evolving +## @replaceable no +function ctest_initialize +{ + # shellcheck disable=SC2034 + CTEST_OUTPUT_ON_FAILURE=true +} + +## @description ctest log processor +## @audience private +## @stability evolving +## @replaceable no +## @param module +## @param logfile +## @param filefragment +## @return status +function ctest_process_tests +{ + # shellcheck disable=SC2034 + declare module=$1 + # shellcheck disable=SC2034 + declare buildlogfile=$2 + declare filefrag=$3 + declare result=0 + declare module_failed_tests + declare filenames + declare shortlog + declare reallog + + filenames=$(find "${BASEDIR}" -type f -name LastTestsFailed.log) + + if [[ -n "${filenames}" ]]; then + for shortlog in ${filenames}; do + #shellcheck disable=2086 + reallog="$(dirname ${shortlog})/LastTest.log" + if [[ -f "${reallog}" ]]; then + module_failed_tests=$(cut -f2 -d: "${shortlog}") + cp "${reallog}" "${PATCH_DIR}/patch-${filefrag}.ctest" + CTEST_LOGS="${CTEST_LOGS} @@BASE@@/patch-${filefrag}.ctest" + CTEST_FAILED_TESTS="${CTEST_FAILED_TESTS} ${module_failed_tests}" + ((result=result+1)) + fi + done + fi + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +## @description cmake test table results +## @audience private +## @stability evolving +## @replaceable no +## @param jdk +function ctest_finalize_results +{ + declare jdk=$1 + + if [[ -n "${CTEST_FAILED_TESTS}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "${jdk}Failed CTEST tests" ${CTEST_FAILED_TESTS} + CTEST_FAILED_TESTS="" + add_footer_table "CTEST logs" "${CTEST_LOGS}" + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.d/findbugs.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/findbugs.sh b/precommit/test-patch.d/findbugs.sh index 4d8dd78..a923cb9 100755 --- a/precommit/test-patch.d/findbugs.sh +++ b/precommit/test-patch.d/findbugs.sh @@ -339,9 +339,7 @@ function findbugs_postinstall offset_clock "${MODULE_STATUS_TIMER[${i}]}" module="${MODULE[${i}]}" - if [[ ${BUILDTOOLCWD} == true ]]; then - pushd "${module}" >/dev/null - fi + buildtool_cwd "${i}" fn=$(module_file_fragment "${module}") @@ -368,9 +366,7 @@ function findbugs_postinstall "${branchxml}" \ "${patchxml}" if [[ $? != 0 ]]; then - if [[ ${BUILDTOOLCWD} == true ]]; then - popd >/dev/null - fi + popd >/dev/null module_status ${i} -1 "" "${module} cannot run computeBugHistory from findbugs" ((result=result+1)) savestop=$(stop_clock) http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.d/gradle.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/gradle.sh b/precommit/test-patch.d/gradle.sh index 84f5a6b..f6724ad 100755 --- a/precommit/test-patch.d/gradle.sh +++ b/precommit/test-patch.d/gradle.sh @@ -52,7 +52,7 @@ function gradle_initialize { if [[ "${BUILDTOOL}" = gradle ]]; then # shellcheck disable=SC2034 - BUILDTOOLCWD=false + BUILDTOOLCWD=basedir fi # we need to do this before docker kicks in http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.d/make.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.d/make.sh b/precommit/test-patch.d/make.sh new file mode 100755 index 0000000..5e80132 --- /dev/null +++ b/precommit/test-patch.d/make.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRCMAKEIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_build_tool make + +MAKE=make +MAKEFILE=Makefile + +## @description make usage hook +## @audience private +## @stability evolving +## @replaceable no +function make_usage +{ + yetus_add_option "--make-cmd=<cmd>" "The 'make' command to use (default: '${MAKE}')" + yetus_add_option "--make-file=<filename>" "The name of the file the make cmd should work on (default: '${MAKEFILE}')" +} + +## @description make argument parser +## @audience private +## @stability evolving +## @param args +function make_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --make-cmd=*) + MAKE=${i#*=} + ;; + --make-file=*) + MAKEFILE=${i#*=} + ;; + --make-use-git-clean) + MAKE_GITCLEAN=true + ;; + esac + done +} + +## @description get the name of the make build filename +## @audience private +## @stability evolving +## @replaceable no +## @return make build file +function make_buildfile +{ + echo "${MAKEFILE}" +} + +## @description get the name of the make binary +## @audience private +## @stability evolving +## @replaceable no +## @return filename +## @param testname +function make_executor +{ + echo "${MAKE} ${MAKE_ARGS[*]}" +} + +## @description make worker +## @audience private +## @stability evolving +## @replaceable no +## @return status +## @param repostatus +## @param test +function make_modules_worker +{ + declare repostatus=$1 + declare tst=$2 + shift 2 + + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=false + + case ${tst} in + compile) + modules_workers "${repostatus}" "${tst}" + ;; + distclean) + if [[ ${MAKE_GITCLEAN} = true ]];then + git clean -x -f -d + else + modules_workers "${repostatus}" distclean clean + fi + ;; + unit) + modules_workers "${repostatus}" test test + ;; + *) + # shellcheck disable=SC2034 + UNSUPPORTED_TEST=true + if [[ ${repostatus} = patch ]]; then + add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin" + fi + yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}" + return 1 + ;; + esac +} + +## @description make module queuer +## @audience private +## @stability evolving +## @replaceable no +function make_builtin_personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + + declare module + + yetus_debug "Using builtin personality_modules" + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + for module in ${CHANGED_MODULES}; do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} + done +} + +## @description make test determiner +## @audience private +## @stability evolving +## @replaceable no +## @param filename +function make_builtin_personality_file_tests +{ + declare filename=$1 + + yetus_debug "Using builtin make personality_file_tests" + + if [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.hh\.in$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.make$ + || ${filename} =~ Makefile$ + ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test unit + fi +} http://git-wip-us.apache.org/repos/asf/yetus/blob/23c00565/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index b0b373c..da91065 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -78,7 +78,7 @@ function setup_defaults DOCKER_ENABLE_PRIVILEGED=true BUILD_NATIVE=${BUILD_NATIVE:-true} - BUILDTOOLCWD=true + BUILDTOOLCWD=module # shellcheck disable=SC2034 CHANGED_MODULES="" @@ -92,6 +92,7 @@ function setup_defaults ISSUE="" TIMER=$(date +"%s") BUILDTOOL=maven + JVM_REQUIRED=true JDK_TEST_LIST="compile javadoc unit" } @@ -1041,7 +1042,6 @@ function find_changed_modules local builddirs local builddir local module - local buildmods local prev_builddir local i=1 local dir @@ -1059,7 +1059,7 @@ function find_changed_modules # Empty string indicates the build system wants to disable module detection if [[ -z ${buildfile} ]]; then - buiddirs="." + builddirs="." else # Now find all the modules that were changed for i in ${changed_dirs}; do @@ -1081,6 +1081,7 @@ function find_changed_modules #shellcheck disable=SC2086,SC2034 CHANGED_MODULES=$(echo ${builddirs} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u) + #shellcheck disable=SC2086,SC2034 CHANGED_MODULES=$(echo ${CHANGED_MODULES} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u) # turn it back into a list so that anyone printing doesn't @@ -1095,6 +1096,12 @@ function find_changed_modules yetus_debug "Only one entry, so keeping it ${CHANGED_MODULES}" # shellcheck disable=SC2034 CHANGED_UNION_MODULES=${CHANGED_MODULES} + + # some build tools may want to change these and/or + # make other changes based upon these results + if declare -f "${BUILDTOOL}_changed_modules" >/dev/null; then + "${BUILDTOOL}_changed_modules" + fi return fi @@ -1123,6 +1130,8 @@ function find_changed_modules #shellcheck disable=SC2034 CHANGED_UNION_MODULES="${builddir}" + # some build tools may want to change these and/or + # make other changes based upon these results if declare -f ${BUILDTOOL}_reorder_modules >/dev/null; then "${BUILDTOOL}_reorder_modules" "${repostatus}" fi @@ -1482,6 +1491,40 @@ function copytpbits popd >/dev/null } +## @description change the working directory to execute the buildtool +## @audience public +## @stability evolving +## @replaceable no +## @param MODULE_ index +function buildtool_cwd +{ + declare modindex=$1 + + BUILDTOOLCWD="${BUILDTOOLCWD//@@@BASEDIR@@@/${BASEDIR}}" + BUILDTOOLCWD="${BUILDTOOLCWD//@@@MODULEDIR@@@/${BASEDIR}/${MODULE[${modindex}]}}" + + if [[ "${BUILDTOOLCWD}" =~ ^/ ]]; then + yetus_debug "buildtool_cwd: ${BUILDTOOLCWD}" + if [[ ! -e "${BUILDTOOLCWD}" ]]; then + mkdir -p "${BUILDTOOLCWD}" + fi + pushd "${BUILDTOOLCWD}" >/dev/null + return $? + fi + + case "${BUILDTOOLCWD}" in + basedir) + pushd "${BASEDIR}" >/dev/null + ;; + module) + pushd "${BASEDIR}/${MODULE[${modindex}]}" >/dev/null + ;; + *) + pushd "$(pwd)" + ;; + esac +} + ## @description If this patches actually patches test-patch.sh, then ## @description run with the patched version for the test. ## @audience private @@ -1735,19 +1778,20 @@ function module_status ## @param mvncmdline function modules_workers { - local repostatus=$1 - local testtype=$2 + declare repostatus=$1 + declare testtype=$2 shift 2 - local modindex=0 - local fn - local savestart=${TIMER} - local savestop - local repo - local modulesuffix - local jdk="" - local jdkindex=0 - local statusjdk - local result=0 + declare modindex=0 + declare fn + declare savestart=${TIMER} + declare savestop + declare repo + declare modulesuffix + declare jdk="" + declare jdkindex=0 + declare statusjdk + declare result=0 + declare argv if [[ ${repostatus} == branch ]]; then repo=${PATCH_BRANCH} @@ -1772,9 +1816,7 @@ function modules_workers fn=$(module_file_fragment "${MODULE[${modindex}]}") fn="${fn}${jdk}" modulesuffix=$(basename "${MODULE[${modindex}]}") - if [[ ${BUILDTOOLCWD} == true ]]; then - pushd "${BASEDIR}/${MODULE[${modindex}]}" >/dev/null - fi + buildtool_cwd "${modindex}" if [[ ${modulesuffix} == . ]]; then modulesuffix="root" @@ -1786,11 +1828,14 @@ function modules_workers continue fi + argv=("${@//@@@MODULEFN@@@/${fn}}") + argv=("${argv[@]//@@@MODULEDIR@@@/${BASEDIR}/${MODULE[${modindex}]}}") + # shellcheck disable=2086,2046 echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \ - $("${BUILDTOOL}_executor") \ + $("${BUILDTOOL}_executor" "${testtype}") \ ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} \ - "${@//@@@MODULEFN@@@/${fn}}" + "${argv[@]}" if [[ $? == 0 ]] ; then module_status \ @@ -1817,9 +1862,7 @@ function modules_workers MODULE_STATUS_TIMER[${modindex}]=${savestop} # shellcheck disable=SC2086 echo "Elapsed: $(clock_display ${savestop})" - if [[ ${BUILDTOOLCWD} == true ]]; then - popd >/dev/null - fi + popd >/dev/null ((modindex=modindex+1)) done @@ -1968,9 +2011,7 @@ function check_unittests fn="${fn}${jdk}" test_logfile="${PATCH_DIR}/patch-unit-${fn}.txt" - if [[ ${BUILDTOOLCWD} == true ]]; then - pushd "${MODULE[${i}]}" >/dev/null - fi + buildtool_cwd "${i}" needlog=0 for testsys in ${TESTFORMATS}; do @@ -1989,9 +2030,7 @@ function check_unittests unitlogs="${unitlogs} @@BASE@@/patch-unit-${fn}.txt" fi - if [[ ${BUILDTOOLCWD} == true ]]; then - popd >/dev/null - fi + popd >/dev/null ((i=i+1)) done @@ -2519,7 +2558,7 @@ function generic_post_handler ## @param branch|patch ## @return 0 on success ## @return 1 on failure -function compile +function compile_jvm { declare codebase=$1 declare result=0 @@ -2527,17 +2566,6 @@ function compile declare multijdkmode=false declare jdkindex=0 - verify_needed_test compile - if [[ $? == 0 ]]; then - return 0 - fi - - if [[ ${codebase} = "branch" ]]; then - big_console_header "Pre-patch ${PATCH_BRANCH} compilation" - else - big_console_header "Patch compilation" - fi - verify_multijdk_test compile if [[ $? == 1 ]]; then multijdkmode=true @@ -2548,18 +2576,7 @@ function compile JAVA_HOME=${jdkindex} fi - personality_modules "${codebase}" compile - "${BUILDTOOL}_modules_worker" "${codebase}" compile - modules_messages "${codebase}" compile true - - for plugin in ${TESTTYPES}; do - verify_patchdir_still_exists - if declare -f ${plugin}_compile >/dev/null 2>&1; then - yetus_debug "Running ${plugin}_compile ${codebase} ${multijdkmode}" - "${plugin}_compile" "${codebase}" "${multijdkmode}" - ((result = result + $?)) - fi - done + compile_nonjvm "${codebase}" "${multijdkmode}" done JAVA_HOME=${savejavahome} @@ -2570,6 +2587,72 @@ function compile return 0 } +## @description Execute the compile phase. This will callout +## @description to _compile +## @audience public +## @stability evolving +## @replaceable no +## @param branch|patch +## @return 0 on success +## @return 1 on failure +function compile_nonjvm +{ + declare codebase=$1 + declare result=0 + declare -r savejavahome=${JAVA_HOME} + declare multijdkmode=${2:-false} + declare jdkindex=0 + + personality_modules "${codebase}" compile + "${BUILDTOOL}_modules_worker" "${codebase}" compile + modules_messages "${codebase}" compile true + + for plugin in ${TESTTYPES}; do + verify_patchdir_still_exists + if declare -f ${plugin}_compile >/dev/null 2>&1; then + yetus_debug "Running ${plugin}_compile ${codebase} ${multijdkmode}" + "${plugin}_compile" "${codebase}" "${multijdkmode}" + ((result = result + $?)) + fi + done + + if [[ ${result} -gt 0 ]]; then + return 1 + fi + return 0 +} + +## @description Execute the compile phase. This will callout +## @description to _compile +## @audience public +## @stability evolving +## @replaceable no +## @param branch|patch +## @return 0 on success +## @return 1 on failure +function compile +{ + declare codebase=$1 + + verify_needed_test compile + if [[ $? == 0 ]]; then + return 0 + fi + + if [[ ${codebase} = "branch" ]]; then + big_console_header "Pre-patch ${PATCH_BRANCH} compilation" + else + big_console_header "Patch compilation" + fi + + yetus_debug "Is JVM Required? ${JVM_REQUIRED}" + if [[ "${JVM_REQUIRED}" = true ]]; then + compile_jvm "${codebase}" + else + compile_nonjvm "${codebase}" + fi +} + ## @description Execute the static analysis test cycle. ## @description This will callout to _precompile, compile, _postcompile and _rebuild ## @audience public @@ -2669,8 +2752,7 @@ function distclean declare result=0 declare plugin - personality_modules branch distclean - "${BUILDTOOL}_modules_worker" branch distclean + big_console_header "Cleaning the source tree" for plugin in ${TESTTYPES} ${TESTFORMATS}; do if declare -f ${plugin}_clean >/dev/null 2>&1; then @@ -2683,6 +2765,10 @@ function distclean fi done + personality_modules branch distclean + "${BUILDTOOL}_modules_worker" branch distclean + (( result = result + $? )) + if [[ ${result} -gt 0 ]]; then return 1 fi @@ -2723,6 +2809,9 @@ function initialize fi plugins_initialize + if [[ ${RESULT} != 0 ]]; then + cleanup_and_exit 1 + fi locate_patch
