Repository: yetus Updated Branches: refs/heads/master 87014f42a -> 577e74f2c
YETUS-123. Various Docker mode fixes: plug-in issues, local patch file problems, availability, & Dockerfile re-usability Signed-off-by: Kengo Seki <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/577e74f2 Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/577e74f2 Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/577e74f2 Branch: refs/heads/master Commit: 577e74f2ca59222219f02e5d86fc4c34e3f8c39d Parents: 87014f4 Author: Allen Wittenauer <[email protected]> Authored: Thu Nov 12 08:21:21 2015 -0800 Committer: Allen Wittenauer <[email protected]> Committed: Tue Nov 17 21:10:28 2015 -0800 ---------------------------------------------------------------------- .../in-progress/precommit-advanced.md | 4 +- precommit/core.d/01-common.sh | 19 +- precommit/core.d/docker.sh | 506 +++++++++++++++++++ precommit/core.d/patchfiles.sh | 5 + precommit/test-patch-docker/Dockerfile | 99 ++++ precommit/test-patch-docker/Dockerfile-endstub | 2 + .../test-patch-docker/Dockerfile-startstub | 91 ---- .../test-patch-docker/launch-test-patch.sh | 13 +- .../test-patch-docker/test-patch-docker.sh | 392 -------------- precommit/test-patch.sh | 121 ++--- 10 files changed, 697 insertions(+), 555 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/asf-site-src/source/documentation/in-progress/precommit-advanced.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/in-progress/precommit-advanced.md b/asf-site-src/source/documentation/in-progress/precommit-advanced.md index 638f7d2..dda9546 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-advanced.md +++ b/asf-site-src/source/documentation/in-progress/precommit-advanced.md @@ -27,9 +27,9 @@ test-patch # Docker Support -By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself into a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java. +By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself in a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a Jenkins instance. -The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. However be aware that test-patch will copy this file and append its necessary hooks to re-launch itself prior to executing docker. +The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. test-patch will copy this file up until the text "YETUS CUT HERE" to a different directory and then append its necessary hooks to re-launch itself prior to executing docker. NOTE: If you are using Boot2Docker, you must use directories under /Users (OSX) or C:\Users (Windows) as the base and patchprocess directories (specified by the --basedir and --patch-dir options respectively), because automatically mountable directories are limited to them. See [the Docker documentation](https://docs.docker.com/userguide/dockervolumes/#mount-a-host-directory-as-a-data-volume). http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/core.d/01-common.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh index 2110f75..5e7cb60 100755 --- a/precommit/core.d/01-common.sh +++ b/precommit/core.d/01-common.sh @@ -45,6 +45,8 @@ function common_defaults PATCH_METHODS=("gitapply" "patchcmd") #shellcheck disable=SC2034 PATCH_LEVEL=0 + #shellcheck disable=SC2034 + PATCH_SYSTEM="" PROJECT_NAME=yetus RESULT=0 #shellcheck disable=SC2034 @@ -434,22 +436,19 @@ function importplugins files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh) fi + if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then + yetus_error "ERROR: Can't find ${PERSONALITY} to import." + unset PERSONALITY + fi + if [[ -z ${PERSONALITY} && -f "${BINDIR}/personality/${PROJECT_NAME}.sh" && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then + yetus_debug "Using project personality." PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh" fi - if [[ -n ${PERSONALITY} ]]; then - if [[ ! -f ${PERSONALITY} ]]; then - if [[ -f "${BINDIR}/personality/${PROJECT_NAME}.sh" - && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then - PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh" - else - yetus_debug "Can't find ${PERSONALITY} to import." - return - fi - fi + if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then yetus_debug "Importing ${PERSONALITY}" # shellcheck disable=SC1090 . "${PERSONALITY}" http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/core.d/docker.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh new file mode 100755 index 0000000..f121274 --- /dev/null +++ b/precommit/core.d/docker.sh @@ -0,0 +1,506 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DOCKER_ID=${RANDOM} +DOCKER_DESTRUCTIVE=true + +## @description Put docker stats in various tables +## @stability stable +## @audience private +## @replaceable yes +function docker_finish_stats +{ + if [[ ${DOCKERMODE} == true ]]; then + # DOCKER_VERSION is set by our creator. + add_footer_table "Docker" "${DOCKER_VERSION}" + fi +} + +## @description Verify docker exists +## @audience private +## @stability evolving +## @replaceable no +## @returns 1 if docker not found +## @returns 0 if docker is found +function dockerverify +{ + declare pathdocker + + if [[ -z "${DOCKERCMD}" ]]; then + pathdocker=$(which docker 2>/dev/null) + + if [[ ! -f "${pathdocker}" ]]; then + yetus_error "Docker cannot be found." + return 1 + fi + DOCKERCMD="${pathdocker}" + fi + + if [[ ! -x "${DOCKERCMD}" ]];then + yetus_error "Docker command ${DOCKERCMD} is not executable." + return 1 + fi + + ${DOCKERCMD} info >/dev/null 2>&1 + + if [[ $? != 0 ]]; then + yetus_error "Docker is not functioning properly. Daemon down/unreachable?" + return 1 + fi + return 0 +} + +## @description Run docker with some arguments, and +## @description optionally send to debug +## @audience private +## @stability evolving +## @replaceable no +## @param args +function dockercmd +{ + yetus_debug "${DOCKERCMD} $*" + "${DOCKERCMD}" "$@" +} + +## @description Stop and delete all defunct containers +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_stop_exited_containers +{ + declare line + declare id + declare value + declare size + declare exitfn="${PATCH_DIR}/dsec.$$" + + big_console_header "Removing stopped/exited containers" + + echo "Docker containers in exit state:" + + dockercmd ps -a | ${GREP} Exited > "${exitfn}" + if [[ ! -s "${exitfn}" ]]; then + return + fi + + # stop *all* containers that are in exit state for + # more than > 8 hours + while read -r line; do + id=$(echo "${line}" | cut -f1 -d' ') + value=$(echo "${line}" | cut -f2 -d' ') + size=$(echo "${line}" | cut -f3 -d' ') + + if [[ ${size} =~ day + || ${size} =~ week + || ${size} =~ month + || ${size} =~ year ]]; then + echo "Removing docker ${id}" + if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then + dockercmd rm "${id}" + else + echo docker rm "${id}" + fi + fi + + if [[ ${size} =~ hours + && ${value} -gt 8 ]]; then + echo "Removing docker ${id}" + if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then + dockercmd rm "${id}" + else + echo docker rm "${id}" + fi + fi + done < <( + ${SED} -e 's,ago,,g' "${exitfn}"\ + | ${AWK} '{print $1" "$(NF - 2)" "$(NF - 1)}') + rm "${exitfn}" +} + +## @description Remove all containers that are not +## @description are not running + older than 1 day +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_rm_old_containers +{ + declare line + declare id + declare value + declare size + declare running + declare stoptime + declare curtime + + big_console_header "Removing old containers" + + while read -r line; do + id=$(echo "${line}" | cut -f1 -d, ) + running=$(echo "${line}" | cut -f2 -d, ) + stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) + + if [[ ${running} = true ]]; then + yetus_debug "${id} is still running, skipping." + continue + fi + + # believe it or not, date is not even close to standardized... + if [[ $(uname -s) == Linux ]]; then + + # GNU date + stoptime=$(date -d "${stoptime}" "+%s") + else + + # BSD date + stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s") + fi + + curtime=$(date "+%s") + ((difftime = curtime - stoptime)) + if [[ ${difftime} -gt 86400 ]]; then + echo "Removing docker ${id}" + if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then + dockercmd rm "${id}" + else + echo docker rm "${id}" + fi + fi + done < <( + # see https://github.com/koalaman/shellcheck/issues/375 + # shellcheck disable=SC2046 + dockercmd inspect \ + -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \ + $(dockercmd ps -qa) 2>/dev/null) +} + +## @description Remove untagged/unused images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_remove_untagged_images +{ + + big_console_header "Removing untagged images" + + # this way is a bit more compatible with older docker versions + # shellcheck disable=SC2016 + dockercmd images | tail -n +2 | ${AWK} '$1 == "<none>" {print $3}' | \ + xargs --no-run-if-empty docker rmi +} + +## @description Remove defunct tagged images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_remove_old_tagged_images +{ + declare line + declare id + declare created + + big_console_header "Removing old tagged images" + + while read -r line; do + # shellcheck disable=SC2016 + id=$(echo "${line}" | ${AWK} '{print $1":"$2}') + # shellcheck disable=SC2016 + created=$(echo "${line}" | ${AWK} '{print $5}') + + if [[ ${created} =~ week + || ${created} =~ month + || ${created} =~ year ]]; then + echo "Removing docker image ${id}" + if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then + dockercmd rmi "${id}" + else + echo docker rmi "${id}" + fi + fi + + if [[ ${id} =~ yetus/${PROJECT_NAME}:date + || ${id} =~ test-patch- ]]; then + if [[ ${created} =~ day + || ${created} =~ hours ]]; then + echo "Removing docker image ${id}" + if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then + dockercmd rmi "${id}" + else + echo docker rmi "${id}" + fi + fi + fi + done < <(dockercmd images) +} + +## @description Performance docker maintenance on Jenkins +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_cleanup_apache_jenkins +{ + echo "==========================" + echo "Docker Images:" + dockercmd images + echo "==========================" + echo "Docker Containers:" + dockercmd ps -a + echo "==========================" + + docker_stop_exited_containers + + docker_rm_old_containers + + docker_remove_untagged_images + + docker_remove_old_tagged_images +} + +## @description Clean up our old images used for patch testing +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_cleanup_yetus_images +{ + declare images + declare imagecount + declare rmimage + declare rmi + + # we always want to leave at least one of our images + # so that the whole thing doesn't have to be rebuilt. + # This also let's us purge any old images so that + # we can get fresh stuff sometimes + # shellcheck disable=SC2016 + images=$(dockercmd images | ${GREP} "yetus/${PROJECT_NAME}" | ${GREP} tp | ${AWK} '{print $1":"$2}') 2>&1 + + # shellcheck disable=SC2086 + imagecount=$(echo ${images} | tr ' ' '\n' | wc -l) + ((imagecount = imagecount - 1 )) + + # shellcheck disable=SC2086 + rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount}) + for rmi in ${rmimage} + do + echo "Removing image ${rmi}" + if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then + dockercmd rmi "${rmi}" + else + echo docker rmi "${rmi}" + fi + done +} + +## @description Perform pre-run maintenance to free up +## @description resources. With --jenkins, it is a lot +## @description more destructive. +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_cleanup +{ + if [[ ${TESTPATCHMODE} =~ jenkins ]]; then + docker_cleanup_apache_jenkins + fi + + docker_cleanup_yetus_images +} + +## @description Deterine the user name and user id of the user +## @description that the docker container should use +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_determine_user +{ + # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some + # ... person ... sets it to an account that doesn't actually exist. + # so instead, we need to try and override it with something that's + # probably close to reality. + if [[ ${TESTPATCHMODE} =~ jenkins ]]; then + USER=$(id | cut -f2 -d\( | cut -f1 -d\)) + fi + + if [[ "$(uname -s)" == "Linux" ]]; then + USER_NAME=${SUDO_USER:=$USER} + USER_ID=$(id -u "${USER_NAME}") + GROUP_ID=$(id -g "${USER_NAME}") + else # boot2docker uid and gid + USER_NAME=${USER} + USER_ID=1000 + GROUP_ID=50 + fi +} + +## @description Determine the revision of a dockerfile +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_getfilerev +{ + ${GREP} 'YETUS_PRIVATE: gitrev=' \ + "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ + | cut -f2 -d= +} + +function docker_version +{ + declare vertype=$1 + declare val + + + # new version command + val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null) + + if [[ $? != 0 ]];then + # old version command + val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ') + fi + + echo "${val}" +} + +## @description Start a test patch docker container +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_run_image +{ + declare dockerfilerev + declare baseimagename + declare patchimagename="yetus/${PROJECT_NAME}:tp-${DOCKER_ID}" + declare client + declare server + declare retval + declare elapsed + + dockerfilerev=$(docker_getfilerev) + + baseimagename="yetus/${PROJECT_NAME}:${dockerfilerev}" + + # make a base image, if it isn't available + big_console_header "Building base image: ${baseimagename}" + start_clock + dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker" + retval=$? + + #shellcheck disable=SC2046 + elapsed=$(clock_display $(stop_clock)) + + echo "" + echo "Total Elapsed time: ${elapsed}" + echo "" + + if [[ ${retval} != 0 ]]; then + yetus_error "ERROR: Docker failed to build image." + add_vote_table -1 docker "Docker failed to build ${baseimagename}." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + + big_console_header "Building patch image: ${patchimagename}" + start_clock + # using the base image, make one that is patch specific + dockercmd build -t "${patchimagename}" - <<PatchSpecificDocker +FROM ${baseimagename} +RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} +RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} +RUN chown -R ${USER_NAME} /home/${USER_NAME} +ENV HOME /home/${USER_NAME} +USER ${USER_NAME} +PatchSpecificDocker + + retval=$? + + #shellcheck disable=SC2046 + elapsed=$(clock_display $(stop_clock)) + + echo "" + echo "Total Elapsed time: ${elapsed}" + echo "" + + if [[ ${retval} != 0 ]]; then + yetus_error "ERROR: Docker failed to build image." + add_vote_table -1 docker "Docker failed to build ${patchimagename}." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + + if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then + extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt") + else + extraargs="" + fi + + client=$(docker_version Client) + server=$(docker_version Server) + + dockerversion="Client=${client} Server=${server}" + + if [[ ${PATCH_DIR} =~ ^/ ]]; then + exec "${DOCKERCMD}" run --rm=true -i \ + ${extraargs} \ + -v "${PWD}:/testptch/${PROJECT_NAME}" \ + -v "${PATCH_DIR}:/testptch/patchprocess" \ + -u "${USER_NAME}" \ + -w "/testptch/${PROJECT_NAME}" \ + --env=BASEDIR="/testptch/${PROJECT_NAME}" \ + --env=DOCKER_VERSION="${dockerversion} Image:${baseimagename}" \ + --env=JAVA_HOME="${JAVA_HOME}" \ + --env=PATCH_DIR=/testptch/patchprocess \ + --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \ + --env=PROJECT_NAME="${PROJECT_NAME}" \ + --env=TESTPATCHMODE="${TESTPATCHMODE}" \ + "${patchimagename}" + else + exec "${DOCKERCMD}" run --rm=true -i \ + ${extraargs} \ + -v "${PWD}:/testptch/${PROJECT_NAME}" \ + -u "${USER_NAME}" \ + -w "/testptch/${PROJECT_NAME}" \ + --env=BASEDIR="/testptch/${PROJECT_NAME}" \ + --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ + --env=JAVA_HOME="${JAVA_HOME}" \ + --env=PATCH_DIR="${PATCH_DIR}" \ + --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \ + --env=PROJECT_NAME="${PROJECT_NAME}" \ + --env=TESTPATCHMODE="${TESTPATCHMODE}" \ + "${patchimagename}" + fi +} + +## @description Switch over to a Docker container +## @audience private +## @stability evolving +## @replaceable no +## @param args +function docker_handler +{ + PATCH_DIR=$(relative_dir "${PATCH_DIR}") + + docker_cleanup + docker_determine_user + docker_run_image +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/core.d/patchfiles.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/patchfiles.sh b/precommit/core.d/patchfiles.sh index 7bb2f8b..7154a58 100755 --- a/precommit/core.d/patchfiles.sh +++ b/precommit/core.d/patchfiles.sh @@ -97,6 +97,7 @@ function locate_patch # it's a declarely provided file if [[ -f ${PATCH_OR_ISSUE} ]]; then patchfile="${PATCH_OR_ISSUE}" + PATCH_SYSTEM=generic else # run through the bug systems. maybe they know? for bugsys in ${BUGSYSTEMS}; do @@ -104,6 +105,7 @@ function locate_patch "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" if [[ $? == 0 ]]; then gotit=true + PATCH_SYSTEM=${bugsys} fi fi done @@ -115,9 +117,12 @@ function locate_patch yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}." cleanup_and_exit 1 fi + PATCH_SYSTEM=generic fi fi + yetus_debug "Determined patch system to be ${PATCH_SYSTEM}" + if [[ ! -f "${PATCH_DIR}/patch" && -f "${patchfile}" ]]; then cp "${patchfile}" "${PATCH_DIR}/patch" http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/test-patch-docker/Dockerfile ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile new file mode 100644 index 0000000..ee92c33 --- /dev/null +++ b/precommit/test-patch-docker/Dockerfile @@ -0,0 +1,99 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:trusty + +WORKDIR /root + +###### +# Install common dependencies from packages +###### +RUN apt-get update && apt-get install --no-install-recommends -y \ + git curl ant make maven \ + cmake gcc g++ protobuf-compiler \ + build-essential libtool \ + zlib1g-dev pkg-config libssl-dev \ + snappy libsnappy-dev \ + bzip2 libbz2-dev \ + libjansson-dev \ + fuse libfuse-dev \ + libcurl4-openssl-dev \ + python python2.7 pylint \ + ruby \ + openjdk-7-jdk \ + libperl-critic-perl + +# Fixing the Apache commons / Maven dependency problem under Ubuntu: +# See http://wiki.apache.org/commons/VfsProblems +RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . + +####### +# Oracle Java +####### + +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:webupd8team/java +RUN apt-get update + + +# Auto-accept the Oracle JDK license +RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +RUN apt-get install -y oracle-java7-installer + +# Auto-accept the Oracle JDK license +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +RUN apt-get install -y oracle-java8-installer + +###### +# Install findbugs +###### +RUN mkdir -p /opt/findbugs && \ + curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ + -o /opt/findbugs.tar.gz && \ + tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs +ENV FINDBUGS_HOME /opt/findbugs + +#### +# Install shellcheck +#### +RUN apt-get install -y cabal-install +RUN cabal update && cabal install shellcheck --global + +#### +# Install rubocop +### +RUN gem install rubocop --no-ri --no-rdoc + +#### +# Install ruby-lint +### +RUN gem install ruby-lint --no-ri --no-rdoc + +#### +# Install bats +#### +RUN add-apt-repository -y ppa:duggan/bats +RUN apt-get update +RUN apt-get install -y bats + +### +# Set the locale +### +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/test-patch-docker/Dockerfile-endstub ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile-endstub b/precommit/test-patch-docker/Dockerfile-endstub index aa0463e..ce76491 100644 --- a/precommit/test-patch-docker/Dockerfile-endstub +++ b/precommit/test-patch-docker/Dockerfile-endstub @@ -15,5 +15,7 @@ # limitations under the License. ADD launch-test-patch.sh /testptch/launch-test-patch.sh +RUN mkdir /testptch/extras +RUN chmod a+rwx /testptch/extras RUN chmod a+rx /testptch/launch-test-patch.sh CMD /testptch/launch-test-patch.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/test-patch-docker/Dockerfile-startstub ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile-startstub b/precommit/test-patch-docker/Dockerfile-startstub deleted file mode 100644 index d191285..0000000 --- a/precommit/test-patch-docker/Dockerfile-startstub +++ /dev/null @@ -1,91 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:trusty - -WORKDIR /root - -###### -# Install common dependencies from packages -###### -RUN apt-get update && apt-get install --no-install-recommends -y \ - git curl ant make maven \ - cmake gcc g++ protobuf-compiler \ - build-essential libtool \ - zlib1g-dev pkg-config libssl-dev \ - snappy libsnappy-dev \ - bzip2 libbz2-dev \ - libjansson-dev \ - fuse libfuse-dev \ - libcurl4-openssl-dev \ - python python2.7 pylint \ - ruby \ - openjdk-7-jdk \ - libperl-critic-perl - -# Fixing the Apache commons / Maven dependency problem under Ubuntu: -# See http://wiki.apache.org/commons/VfsProblems -RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . - -####### -# Oracle Java -####### - -RUN apt-get install -y software-properties-common -RUN add-apt-repository -y ppa:webupd8team/java -RUN apt-get update - - -# Auto-accept the Oracle JDK license -RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get install -y oracle-java7-installer - -# Auto-accept the Oracle JDK license -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get install -y oracle-java8-installer - -###### -# Install findbugs -###### -RUN mkdir -p /opt/findbugs && \ - curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ - -o /opt/findbugs.tar.gz && \ - tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs -ENV FINDBUGS_HOME /opt/findbugs - -#### -# Install shellcheck -#### -RUN apt-get install -y cabal-install -RUN cabal update && cabal install shellcheck --global - -#### -# Install rubocop -### -RUN gem install rubocop - -#### -# Install ruby-lint -### -RUN gem install ruby-lint - -#### -# Install bats -#### -RUN add-apt-repository -y ppa:duggan/bats -RUN apt-get update -RUN apt-get install -y bats http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/test-patch-docker/launch-test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/launch-test-patch.sh b/precommit/test-patch-docker/launch-test-patch.sh index ac976ad..e7fb4dd 100755 --- a/precommit/test-patch-docker/launch-test-patch.sh +++ b/precommit/test-patch-docker/launch-test-patch.sh @@ -38,6 +38,15 @@ TESTPATCHMODE=${TESTPATCHMODE/--docker } cd "${BASEDIR}" PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P) +# if patch system is generic, then it's either a local +# patch file or was in some other way not pulled from a bug +# system. So we need to rescue it and then tell +# test-patch where to find it. +if [[ "${PATCH_SYSTEM}" = generic ]]; then + cp -p "${PATCH_DIR}/patch" /testptch/extras/patch + patchfile="/testptch/extras/patch" +fi + cd "${PATCH_DIR}/precommit/" #shellcheck disable=SC2086 "${PATCH_DIR}/precommit/test-patch.sh" \ @@ -46,5 +55,5 @@ cd "${PATCH_DIR}/precommit/" --basedir="${BASEDIR}" \ --patch-dir="${PATCH_DIR}" \ --java-home="${JAVA_HOME}" \ - --personality="${PATCH_DIR}/precommit/personality/provided.sh" \ - --user-plugins="${PATCH_DIR}/precommit/user-plugins" \ No newline at end of file + --user-plugins="${PATCH_DIR}/precommit/user-plugins" \ + ${patchfile} http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/test-patch-docker/test-patch-docker.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/test-patch-docker.sh b/precommit/test-patch-docker/test-patch-docker.sh deleted file mode 100755 index ca910ff..0000000 --- a/precommit/test-patch-docker/test-patch-docker.sh +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -DID=${RANDOM} - -## @description Print a message to stderr if --debug is turned on -## @audience private -## @stability stable -## @replaceable no -## @param string -function yetus_debug -{ - if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then - echo "[$(date) DEBUG]: $*" 1>&2 - fi -} - -## @description Run docker with some arguments, and -## @description optionally send to debug -## @audience private -## @stability evolving -## @replaceable no -## @param args -function dockercmd -{ - yetus_debug "docker $*" - docker "$@" -} - -## @description Handle command line arguments -## @audience private -## @stability evolving -## @replaceable no -## @param args -function parse_args -{ - local i - - for i in "$@"; do - case ${i} in - --debug) - YETUS_SHELL_SCRIPT_DEBUG=true - ;; - --dockerversion=*) - DOCKER_VERSION=${i#*=} - ;; - --help|-help|-h|help|--h|--\?|-\?|\?) - yetus_usage - exit 0 - ;; - --java-home=*) - JAVA_HOME=${i#*=} - ;; - --patch-dir=*) - PATCH_DIR=${i#*=} - ;; - --project=*) - PROJECT_NAME=${i#*=} - ;; - *) - ;; - esac - done -} - -## @description Stop and delete all defunct containers -## @audience private -## @stability evolving -## @replaceable no -## @param args -function stop_exited_containers -{ - local line - local id - local value - local size - - echo "Docker containers in exit state:" - - dockercmd ps -a | grep Exited - - # stop *all* containers that are in exit state for - # more than > 8 hours - while read -r line; do - id=$(echo "${line}" | cut -f1 -d' ') - value=$(echo "${line}" | cut -f2 -d' ') - size=$(echo "${line}" | cut -f3 -d' ') - - if [[ ${size} =~ day - || ${size} =~ week - || ${size} =~ month - || ${size} =~ year ]]; then - echo "Removing docker ${id}" - dockercmd rm "${id}" - fi - - if [[ ${size} =~ hours - && ${value} -gt 8 ]]; then - echo "Removing docker ${id}" - dockercmd rm "${id}" - fi - done < <( - dockercmd ps -a \ - | grep Exited \ - | sed -e 's,ago,,g' \ - | awk '{print $1" "$(NF - 2)" "$(NF - 1)}') -} - -## @description Remove all containers that are not -## @description are not running + older than 1 day -## @audience private -## @stability evolving -## @replaceable no -## @param args -function rm_old_containers -{ - declare line - declare id - declare value - declare size - declare running - - while read -r line; do - id=$(echo "${line}" | cut -f1 -d, ) - running=$(echo "${line}" | cut -f2 -d, ) - stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) - - if [[ ${running} = true ]]; then - yetus_debug "${id} is still running, skipping." - continue - fi - - # believe it or not, date is not even close to standardized... - if [[ $(uname -s) == Linux ]]; then - - # GNU date - stoptime=$(date -d "${stoptime}" "+%s") - else - - # BSD date - stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s") - fi - - curtime=$(date "+%s") - ((difftime = curtime - stoptime)) - if [[ ${difftime} -gt 86400 ]]; then - echo "Removing docker ${id}" - dockercmd rm "${id}" - fi - done < <( - # see https://github.com/koalaman/shellcheck/issues/375 - # shellcheck disable=SC2046 - dockercmd inspect \ - -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \ - $(dockercmd ps -qa) 2>/dev/null) -} - -## @description Remove untagged/unused images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function remove_untagged_images -{ - # this way is a bit more compatible with older docker versions - dockercmd images | tail -n +2 | awk '$1 == "<none>" {print $3}' | \ - xargs --no-run-if-empty docker rmi -} - -## @description Remove defunct tagged images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function remove_old_tagged_images -{ - local line - local id - local created - - while read -r line; do - id=$(echo "${line}" | awk '{print $1}') - created=$(echo "${line}" | awk '{print $5}') - - if [[ ${created} =~ week - || ${created} =~ month - || ${created} =~ year ]]; then - echo "Removing docker image ${id}" - dockercmd rmi "${id}" - fi - - if [[ ${id} =~ test-patch-base-${PROJECT_NAME}-date ]]; then - if [[ ${created} =~ day - || ${created} =~ hours ]]; then - echo "Removing docker image ${id}" - dockercmd rmi "${id}" - fi - fi - done < <(dockercmd images) - -} - -## @description Performance docker maintenance on Jenkins -## @audience private -## @stability evolving -## @replaceable no -## @param args -function cleanup_apache_jenkins_docker -{ - echo "==========================" - echo "Docker Images:" - dockercmd images - echo "==========================" - echo "Docker Containers:" - dockercmd ps -a - echo "==========================" - - stop_exited_containers - - rm_old_containers - - remove_untagged_images - - remove_old_tagged_images -} - -## @description Clean up our old images used for patch testing -## @audience private -## @stability evolving -## @replaceable no -## @param args -function cleanup_test_patch_images -{ - local images - local imagecount - local rmimage - local rmi - - # we always want to leave at least one of our images - # so that the whole thing doesn't have to be rebuilt. - # This also let's us purge any old images so that - # we can get fresh stuff sometimes - images=$(dockercmd images | grep --color=none "test-patch-tp-${PROJECT_NAME}" | awk '{print $1}') 2>&1 - - # shellcheck disable=SC2086 - imagecount=$(echo ${images} | tr ' ' '\n' | wc -l) - ((imagecount = imagecount - 1 )) - - # shellcheck disable=SC2086 - rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount}) - for rmi in ${rmimage} - do - echo "Removing image ${rmi}" - dockercmd rmi "${rmi}" - done -} - -## @description Perform pre-run maintenance to free up -## @description resources. With --jenkins, it is a lot -## @description more destructive. -## @audience private -## @stability evolving -## @replaceable no -## @param args -function cleanup -{ - if [[ ${TESTPATCHMODE} =~ jenkins ]]; then - cleanup_apache_jenkins_docker - fi - - cleanup_test_patch_images -} - -## @description Deterine the user name and user id of the user -## @description that the docker container should use -## @audience private -## @stability evolving -## @replaceable no -## @param args -function determine_user -{ - # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some - # ... person ... sets it to an account that doesn't actually exist. - # so instead, we need to try and override it with something that's - # probably close to reality. - if [[ ${TESTPATCHMODE} =~ jenkins ]]; then - USER=$(id | cut -f2 -d\( | cut -f1 -d\)) - fi - - if [[ "$(uname -s)" == "Linux" ]]; then - USER_NAME=${SUDO_USER:=$USER} - USER_ID=$(id -u "${USER_NAME}") - GROUP_ID=$(id -g "${USER_NAME}") - else # boot2docker uid and gid - USER_NAME=${USER} - USER_ID=1000 - GROUP_ID=50 - fi -} - -## @description Determine the revision of a dockerfile -## @audience private -## @stability evolving -## @replaceable no -## @param args -function getdockerfilerev -{ - grep 'TEST_PATCH_PRIVATE: gitrev=' \ - "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ - | cut -f2 -d= -} - -## @description Start a test patch docker container -## @audience private -## @stability evolving -## @replaceable no -## @param args -function run_image -{ - local dockerfilerev - local baseimagename - - dockerfilerev=$(getdockerfilerev) - - baseimagename="test-patch-base-${PROJECT_NAME}-${dockerfilerev}" - - # make a base image, if it isn't available - dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker" - - # using the base image, make one that is patch specific - dockercmd build -t "test-patch-tp-${PROJECT_NAME}-${DID}" - <<PatchSpecificDocker -FROM ${baseimagename} -RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} -RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} -RUN chown -R ${USER_NAME} /home/${USER_NAME} -ENV HOME /home/${USER_NAME} -USER ${USER_NAME} -PatchSpecificDocker - - if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then - extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt") - else - extraargs="" - fi - - if [[ ${PATCH_DIR} =~ ^/ ]]; then - dockercmd run --rm=true -i \ - ${extraargs} \ - -v "${PWD}:/testptch/${PROJECT_NAME}" \ - -v "${PATCH_DIR}:/testptch/patchprocess" \ - -u "${USER_NAME}" \ - -w "/testptch/${PROJECT_NAME}" \ - --env=BASEDIR="/testptch/${PROJECT_NAME}" \ - --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ - --env=JAVA_HOME="${JAVA_HOME}" \ - --env=PATCH_DIR=/testptch/patchprocess \ - --env=PROJECT_NAME="${PROJECT_NAME}" \ - --env=TESTPATCHMODE="${TESTPATCHMODE}" \ - "test-patch-tp-${PROJECT_NAME}-${DID}" - else - dockercmd run --rm=true -i \ - ${extraargs} \ - -v "${PWD}:/testptch/${PROJECT_NAME}" \ - -u "${USER_NAME}" \ - -w "/testptch/${PROJECT_NAME}" \ - --env=BASEDIR="/testptch/${PROJECT_NAME}" \ - --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ - --env=JAVA_HOME="${JAVA_HOME}" \ - --env=PATCH_DIR="${PATCH_DIR}" \ - --env=PROJECT_NAME="${PROJECT_NAME}" \ - --env=TESTPATCHMODE="${TESTPATCHMODE}" \ - "test-patch-tp-${PROJECT_NAME}-${DID}" - fi -} - -parse_args "$@" -cleanup -determine_user -run_image http://git-wip-us.apache.org/repos/asf/yetus/blob/577e74f2/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index 1847848..3c04dd7 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -56,7 +56,7 @@ function setup_defaults { common_defaults - DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub" + DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile" HOW_TO_CONTRIBUTE="https://yetus.apache.org/documentation/latest/precommit-patchnames"; INSTANCE=${RANDOM} RELOCATE_PATCH_DIR=false @@ -299,18 +299,6 @@ function prepopulate_footer add_footer_table "git revision" "${PATCH_BRANCH} / ${gitrev}" } -## @description Put docker stats in various tables -## @stability stable -## @audience private -## @replaceable yes -function finish_docker_stats -{ - if [[ ${DOCKERMODE} == true ]]; then - # DOCKER_VERSION is set by our creator. - add_footer_table "Docker" "${DOCKER_VERSION}" - fi -} - ## @description Put the max memory consumed by maven at the bottom of the table. ## @audience private ## @stability stable @@ -677,6 +665,7 @@ function yetus_usage echo "--debug If set, then output some extra stuff to stderr" echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" echo "--docker Spawn a docker container" + echo "--dockercmd=<file> Command to use as docker executable (default: docker from path)" echo "--dockerfile=<file> Dockerfile fragment to use as the base" echo "--java-home=<path> Set JAVA_HOME (In Docker mode, this should be local to the image)" echo "--linecomments=<bug> Only write line comments to this comma delimited list (defaults to bugcomments)" @@ -762,6 +751,10 @@ function parse_args --docker) DOCKERSUPPORT=true ;; + --dockercmd=*) + #shellcheck disable=SC2034 + DOCKERCMD=${i#*=} + ;; --dockerfile=*) DOCKERFILE=${i#*=} ;; @@ -849,21 +842,18 @@ function parse_args esac done + if [[ "${DOCKERMODE}" = true ]]; then + add_vote_table 0 reexec "Docker mode activated." + elif [[ "${REEXECED}" = true ]]; then + add_vote_table 0 reexec "Precommit patch detected." + fi + if [[ -n ${REEXECLAUNCHTIMER} ]]; then TIMER=${REEXECLAUNCHTIMER}; else start_clock fi - if [[ ${REEXECED} == true - && ${DOCKERMODE} == true ]]; then - add_vote_table 0 reexec "docker + precommit patch detected." - elif [[ ${REEXECED} == true ]]; then - add_vote_table 0 reexec "precommit patch detected." - elif [[ ${DOCKERMODE} == true ]]; then - add_vote_table 0 reexec "docker mode." - fi - if [[ -z "${PATCH_OR_ISSUE}" ]]; then yetus_usage exit 1 @@ -912,6 +902,13 @@ function parse_args GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt" GITUNIDIFFLINES="${PATCH_DIR}/gitdiffunilines.txt" + if [[ "${REEXECED}" = true + && -f "${PATCH_DIR}/precommit/personality/provided.sh" ]]; then + REEXECPERSONALITY="${PERSONALITY}" + PERSONALITY="${PATCH_DIR}/precommit/personality/provided.sh" + fi + + DOCKERFILE=$(yetus_abs "${DOCKERFILE}") } ## @description Locate the build file for a given directory @@ -1363,9 +1360,10 @@ function apply_patch_file ## @replaceable no function copytpbits { - local dockerdir - local dockfile - local person + declare dockerdir + declare dockfile + declare lines + # we need to copy/consolidate all the bits that might have changed # that are considered part of test-patch. This *might* break # things that do off-path includes, but there isn't much we can @@ -1384,6 +1382,7 @@ function copytpbits # copy our entire universe, preserving links, etc. yetus_debug "copying '${BINDIR}' over to '${PATCH_DIR}/precommit'" + # shellcheck disable=SC2164 (cd "${BINDIR}"; tar cpf - . ) | (cd "${PATCH_DIR}/precommit"; tar xpf - ) if [[ -n "${USER_PLUGIN_DIR}" @@ -1414,12 +1413,18 @@ function copytpbits gitfilerev="date${gitfilerev}" fi ( - echo "### TEST_PATCH_PRIVATE: dockerfile=${DOCKERFILE}" - echo "### TEST_PATCH_PRIVATE: gitrev=${gitfilerev}" - cat "${DOCKERFILE}" + echo "### YETUS_PRIVATE: dockerfile=${DOCKERFILE}" + echo "### YETUS_PRIVATE: gitrev=${gitfilerev}" + lines=$(${GREP} -n 'YETUS CUT HERE' "${DOCKERFILE}" | cut -f1 -d:) + if [[ -z "${lines}" ]]; then + cat "${DOCKERFILE}" + else + head -n "${lines}" "${DOCKERFILE}" + fi # make sure we put some space between, just in case last # line isn't an empty line or whatever printf "\n\n" + echo "### YETUS_PRIVATE: start test-patch-bootstrap" cat "${BINDIR}/test-patch-docker/Dockerfile-endstub" printf "\n\n" @@ -1438,18 +1443,20 @@ function copytpbits ## @return none; otherwise relaunches function check_reexec { - local commentfile=${PATCH_DIR}/tp.${RANDOM} - local tpdir - local copy=false - local testdir - local person - local debugflag + declare commentfile=${PATCH_DIR}/tp.${RANDOM} + declare tpdir + declare copy=false + declare testdir if [[ ${REEXECED} == true ]]; then big_console_header "Re-exec mode detected. Continuing." return fi + # determine if the patch hits + # any test-patch sensitive bits + # if so, we need to copy the universe + # after patching it (copy=true) for testdir in "${BINDIR}" \ "${PERSONALITY}" \ "${USER_PLUGIN_DIR}" \ @@ -1461,6 +1468,14 @@ function check_reexec fi done + if [[ "${DOCKERSUPPORT}" = true ]]; then + dockerverify + if [[ $? == 1 ]]; then + yetus_error "Docker not found or not executable. Disabling Docker mode." + DOCKERSUPPORT=false + fi + fi + if [[ ${copy} == true ]]; then big_console_header "precommit patch detected" @@ -1489,8 +1504,8 @@ function check_reexec fi if [[ ${DOCKERSUPPORT} == true - && ${copy} == false ]]; then - big_console_header "Re-execing under Docker" + && ${copy} == false ]]; then + big_console_header "Re-execing under Docker" fi # copy our universe @@ -1504,43 +1519,31 @@ function check_reexec "${BUILDTOOL}_docker_support" fi - client=$(docker version | grep 'Client version' | cut -f2 -d: | tr -d ' ') - server=$(docker version | grep 'Server version' | cut -f2 -d: | tr -d ' ') - - dockerversion="Client=${client} Server=${server}" - TESTPATCHMODE="${USER_PARAMS[*]}" if [[ -n "${BUILD_URL}" ]]; then TESTPATCHMODE="--build-url=${BUILD_URL} ${TESTPATCHMODE}" fi + + if [[ -f "${PERSONALITY}" ]]; then + TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}" + fi + TESTPATCHMODE="--tpglobaltimer=${GLOBALTIMER} ${TESTPATCHMODE}" TESTPATCHMODE="--tpreexectimer=${TIMER} ${TESTPATCHMODE}" TESTPATCHMODE="--tpinstance=${INSTANCE} ${TESTPATCHMODE}" - TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}" TESTPATCHMODE="--plugins=${ENABLED_PLUGINS// /,} ${TESTPATCHMODE}" TESTPATCHMODE=" ${TESTPATCHMODE}" export TESTPATCHMODE - patchdir=$(relative_dir "${PATCH_DIR}") - - if [[ ${TP_SHELL_SCRIPT_DEBUG} = true ]]; then - debugflag="--debug" - fi - + #shellcheck disable=SC2164 cd "${BASEDIR}" #shellcheck disable=SC2093 - exec bash "${PATCH_DIR}/precommit/test-patch-docker/test-patch-docker.sh" \ - ${debugflag} \ - --dockerversion="${dockerversion}" \ - --java-home="${JAVA_HOME}" \ - --patch-dir="${patchdir}" \ - --project="${PROJECT_NAME}" - + docker_handler else # if we aren't doing docker, then just call ourselves # but from the new path with the new flags - #shellcheck disable=SC2093 + #shellcheck disable=SC2164 cd "${PATCH_DIR}/precommit/" exec "${PATCH_DIR}/precommit/test-patch.sh" \ "${USER_PARAMS[@]}" \ @@ -2525,7 +2528,9 @@ function initialize plugins_initialize - finish_docker_stats + if declare -f docker_finish_stats >/dev/null; then + docker_finish_stats + fi locate_patch @@ -2564,7 +2569,7 @@ function prechecks declare plugin declare result=0 - for plugin in ${BUILDTOOL} ${TESTTYPES} ${TESTFORMATS}; do + for plugin in ${BUILDTOOL} ${NEEDEDTESTS} ${TESTFORMATS}; do verify_patchdir_still_exists if declare -f ${plugin}_precheck >/dev/null 2>&1; then
