Repository: yetus Updated Branches: refs/heads/YETUS-83 2ae51492c -> 23a1c9afc
Revert "YETUS-123-05. Various Docker mode fixes: plug-in issues, local patch file problems, availability, & Dockerfile re-usability" This reverts commit 2ae51492c6363d4afc57edda3a2ac2289c4a4e13. Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/23a1c9af Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/23a1c9af Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/23a1c9af Branch: refs/heads/YETUS-83 Commit: 23a1c9afcf5c893f4d7f8474c1d4a671afd6e42e Parents: 2ae5149 Author: Allen Wittenauer <[email protected]> Authored: Mon Nov 16 15:49:53 2015 -0800 Committer: Allen Wittenauer <[email protected]> Committed: Mon Nov 16 15:49:53 2015 -0800 ---------------------------------------------------------------------- .../documentation/latest/precommit-advanced.md | 4 +- precommit/core.d/01-common.sh | 19 +- precommit/core.d/docker.sh | 408 ------------------- precommit/core.d/patchfiles.sh | 5 - precommit/test-patch-docker/Dockerfile | 99 ----- precommit/test-patch-docker/Dockerfile-endstub | 2 - .../test-patch-docker/Dockerfile-startstub | 91 +++++ .../test-patch-docker/launch-test-patch.sh | 14 +- .../test-patch-docker/test-patch-docker.sh | 392 ++++++++++++++++++ precommit/test-patch.sh | 94 +---- 10 files changed, 516 insertions(+), 612 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/asf-site-src/source/documentation/latest/precommit-advanced.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/latest/precommit-advanced.md b/asf-site-src/source/documentation/latest/precommit-advanced.md index dda9546..638f7d2 100644 --- a/asf-site-src/source/documentation/latest/precommit-advanced.md +++ b/asf-site-src/source/documentation/latest/precommit-advanced.md @@ -27,9 +27,9 @@ test-patch # Docker Support -By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself in a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a Jenkins instance. +By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself into a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java. -The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. test-patch will copy this file up until the text "YETUS CUT HERE" to a different directory and then append its necessary hooks to re-launch itself prior to executing docker. +The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. However be aware that test-patch will copy this file and append its necessary hooks to re-launch itself prior to executing docker. NOTE: If you are using Boot2Docker, you must use directories under /Users (OSX) or C:\Users (Windows) as the base and patchprocess directories (specified by the --basedir and --patch-dir options respectively), because automatically mountable directories are limited to them. See [the Docker documentation](https://docs.docker.com/userguide/dockervolumes/#mount-a-host-directory-as-a-data-volume). http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/core.d/01-common.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh index 5e7cb60..2110f75 100755 --- a/precommit/core.d/01-common.sh +++ b/precommit/core.d/01-common.sh @@ -45,8 +45,6 @@ function common_defaults PATCH_METHODS=("gitapply" "patchcmd") #shellcheck disable=SC2034 PATCH_LEVEL=0 - #shellcheck disable=SC2034 - PATCH_SYSTEM="" PROJECT_NAME=yetus RESULT=0 #shellcheck disable=SC2034 @@ -436,19 +434,22 @@ function importplugins files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh) fi - if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then - yetus_error "ERROR: Can't find ${PERSONALITY} to import." - unset PERSONALITY - fi - if [[ -z ${PERSONALITY} && -f "${BINDIR}/personality/${PROJECT_NAME}.sh" && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then - yetus_debug "Using project personality." PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh" fi - if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then + if [[ -n ${PERSONALITY} ]]; then + if [[ ! -f ${PERSONALITY} ]]; then + if [[ -f "${BINDIR}/personality/${PROJECT_NAME}.sh" + && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then + PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh" + else + yetus_debug "Can't find ${PERSONALITY} to import." + return + fi + fi yetus_debug "Importing ${PERSONALITY}" # shellcheck disable=SC1090 . "${PERSONALITY}" http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/core.d/docker.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh deleted file mode 100755 index d5b0d34..0000000 --- a/precommit/core.d/docker.sh +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -DOCKER_ID=${RANDOM} -DOCKER_DESTRUCTIVE=true - -## @description Run docker with some arguments, and -## @description optionally send to debug -## @audience private -## @stability evolving -## @replaceable no -## @param args -function dockercmd -{ - yetus_debug "docker $*" - "${DOCKERCMD}" "$@" -} - -## @description Stop and delete all defunct containers -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_stop_exited_containers -{ - declare line - declare id - declare value - declare size - declare exitfn="${PATCH_DIR}/dsec.$$" - - big_console_header "Removing stopped/exited containers" - - echo "Docker containers in exit state:" - - dockercmd ps -a | ${GREP} Exited > "${exitfn}" - if [[ ! -s "${exitfn}" ]]; then - return - fi - - # stop *all* containers that are in exit state for - # more than > 8 hours - while read -r line; do - id=$(echo "${line}" | cut -f1 -d' ') - value=$(echo "${line}" | cut -f2 -d' ') - size=$(echo "${line}" | cut -f3 -d' ') - - if [[ ${size} =~ day - || ${size} =~ week - || ${size} =~ month - || ${size} =~ year ]]; then - echo "Removing docker ${id}" - if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then - dockercmd rm "${id}" - else - echo docker rm "${id}" - fi - fi - - if [[ ${size} =~ hours - && ${value} -gt 8 ]]; then - echo "Removing docker ${id}" - if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then - dockercmd rm "${id}" - else - echo docker rm "${id}" - fi - fi - done < <( - cat "${exitfn}" \ - | ${SED} -e 's,ago,,g' \ - | ${AWK} '{print $1" "$(NF - 2)" "$(NF - 1)}') - rm "${exitfn}" -} - -## @description Remove all containers that are not -## @description are not running + older than 1 day -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_rm_old_containers -{ - declare line - declare id - declare value - declare size - declare running - - big_console_header "Removing old containers" - - while read -r line; do - id=$(echo "${line}" | cut -f1 -d, ) - running=$(echo "${line}" | cut -f2 -d, ) - stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) - - if [[ ${running} = true ]]; then - yetus_debug "${id} is still running, skipping." - continue - fi - - # believe it or not, date is not even close to standardized... - if [[ $(uname -s) == Linux ]]; then - - # GNU date - stoptime=$(date -d "${stoptime}" "+%s") - else - - # BSD date - stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s") - fi - - curtime=$(date "+%s") - ((difftime = curtime - stoptime)) - if [[ ${difftime} -gt 86400 ]]; then - echo "Removing docker ${id}" - if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then - dockercmd rm "${id}" - else - echo docker rm "${id}" - fi - fi - done < <( - # see https://github.com/koalaman/shellcheck/issues/375 - # shellcheck disable=SC2046 - dockercmd inspect \ - -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \ - $(dockercmd ps -qa) 2>/dev/null) -} - -## @description Remove untagged/unu${SED} images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_remove_untagged_images -{ - - big_console_header "Removing untagged images" - - # this way is a bit more compatible with older docker versions - dockercmd images | tail -n +2 | ${AWK} '$1 == "<none>" {print $3}' | \ - xargs --no-run-if-empty docker rmi -} - -## @description Remove defunct tagged images -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_remove_old_tagged_images -{ - declare line - declare id - declare created - - big_console_header "Removing old tagged images" - - while read -r line; do - id=$(echo "${line}" | ${AWK} '{print $1":"$2}') - created=$(echo "${line}" | ${AWK} '{print $5}') - - if [[ ${created} =~ week - || ${created} =~ month - || ${created} =~ year ]]; then - echo "Removing docker image ${id}" - if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then - dockercmd rmi "${id}" - else - echo docker rmi "${id}" - fi - fi - - if [[ ${id} =~ yetus/${PROJECT_NAME}:date - || ${id} =~ test-patch- ]]; then - if [[ ${created} =~ day - || ${created} =~ hours ]]; then - echo "Removing docker image ${id}" - if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then - dockercmd rmi "${id}" - else - echo docker rmi "${id}" - fi - fi - fi - done < <(dockercmd images) -} - -## @description Performance docker maintenance on Jenkins -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_cleanup_apache_jenkins -{ - echo "==========================" - echo "Docker Images:" - dockercmd images - echo "==========================" - echo "Docker Containers:" - dockercmd ps -a - echo "==========================" - - docker_stop_exited_containers - - docker_rm_old_containers - - docker_remove_untagged_images - - docker_remove_old_tagged_images -} - -## @description Clean up our old images u${SED} for patch testing -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_cleanup_yetus_images -{ - declare images - declare imagecount - declare rmimage - declare rmi - - # we always want to leave at least one of our images - # so that the whole thing doesn't have to be rebuilt. - # This also let's us purge any old images so that - # we can get fresh stuff sometimes - images=$(dockercmd images | ${GREP} "yetus/${PROJECT_NAME}" | ${GREP} tp | ${AWK} '{print $1":"$2}') 2>&1 - - # shellcheck disable=SC2086 - imagecount=$(echo ${images} | tr ' ' '\n' | wc -l) - ((imagecount = imagecount - 1 )) - - # shellcheck disable=SC2086 - rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount}) - for rmi in ${rmimage} - do - echo "Removing image ${rmi}" - if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then - dockercmd rmi "${rmi}" - else - echo docker rmi "${rmi}" - fi - done -} - -## @description Perform pre-run maintenance to free up -## @description resources. With --jenkins, it is a lot -## @description more destructive. -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_cleanup -{ - if [[ ${TESTPATCHMODE} =~ jenkins ]]; then - docker_cleanup_apache_jenkins - fi - - docker_cleanup_yetus_images -} - -## @description Deterine the user name and user id of the user -## @description that the docker container should use -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_determine_user -{ - # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some - # ... person ... sets it to an account that doesn't actually exist. - # so instead, we need to try and override it with something that's - # probably close to reality. - if [[ ${TESTPATCHMODE} =~ jenkins ]]; then - USER=$(id | cut -f2 -d\( | cut -f1 -d\)) - fi - - if [[ "$(uname -s)" == "Linux" ]]; then - USER_NAME=${SUDO_USER:=$USER} - USER_ID=$(id -u "${USER_NAME}") - GROUP_ID=$(id -g "${USER_NAME}") - else # boot2docker uid and gid - USER_NAME=${USER} - USER_ID=1000 - GROUP_ID=50 - fi -} - -## @description Determine the revision of a dockerfile -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_getfilerev -{ - ${GREP} 'YETUS_PRIVATE: gitrev=' \ - "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ - | cut -f2 -d= -} - -## @description Start a test patch docker container -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_run_image -{ - declare dockerfilerev - declare baseimagename - declare patchimagename="yetus/${PROJECT_NAME}:tp-${DOCKER_ID}" - - dockerfilerev=$(docker_getfilerev) - - baseimagename="yetus/${PROJECT_NAME}:${dockerfilerev}" - - # make a base image, if it isn't available - big_console_header "Building base image: ${baseimagename}" - dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker" - - if [[ $? != 0 ]]; then - yetus_error "ERROR: Docker failed to build image." - add_vote_table -1 docker "Docker failed to build ${baseimagename}." - bugsystem_finalreport 1 - cleanup_and_exit 1 - fi - - big_console_header "Building patch image: ${patchimagename}" - # using the base image, make one that is patch specific - dockercmd build -t "${patchimagename}" - <<PatchSpecificDocker -FROM ${baseimagename} -RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} -RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} -RUN chown -R ${USER_NAME} /home/${USER_NAME} -ENV HOME /home/${USER_NAME} -USER ${USER_NAME} -PatchSpecificDocker - - if [[ $? != 0 ]]; then - yetus_error "ERROR: Docker failed to build image." - add_vote_table -1 docker "Docker failed to build ${patchimagename}." - bugsystem_finalreport 1 - cleanup_and_exit 1 - fi - - if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then - extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt") - else - extraargs="" - fi - - if [[ ${PATCH_DIR} =~ ^/ ]]; then - exec "${DOCKERCMD}" run --rm=true -i \ - ${extraargs} \ - -v "${PWD}:/testptch/${PROJECT_NAME}" \ - -v "${PATCH_DIR}:/testptch/patchprocess" \ - -u "${USER_NAME}" \ - -w "/testptch/${PROJECT_NAME}" \ - --env=BASEDIR="/testptch/${PROJECT_NAME}" \ - --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ - --env=JAVA_HOME="${JAVA_HOME}" \ - --env=PATCH_DIR=/testptch/patchprocess \ - --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \ - --env=PROJECT_NAME="${PROJECT_NAME}" \ - --env=TESTPATCHMODE="${TESTPATCHMODE}" \ - "${patchimagename}" - else - exec "${DOCKERCMD}" run --rm=true -i \ - ${extraargs} \ - -v "${PWD}:/testptch/${PROJECT_NAME}" \ - -u "${USER_NAME}" \ - -w "/testptch/${PROJECT_NAME}" \ - --env=BASEDIR="/testptch/${PROJECT_NAME}" \ - --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ - --env=JAVA_HOME="${JAVA_HOME}" \ - --env=PATCH_DIR="${PATCH_DIR}" \ - --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \ - --env=PROJECT_NAME="${PROJECT_NAME}" \ - --env=TESTPATCHMODE="${TESTPATCHMODE}" \ - "${patchimagename}" - fi -} - -## @description Switch over to a Docker container -## @audience private -## @stability evolving -## @replaceable no -## @param args -function docker_handler -{ - docker_cleanup - docker_determine_user - docker_run_image -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/core.d/patchfiles.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/patchfiles.sh b/precommit/core.d/patchfiles.sh index 7154a58..7bb2f8b 100755 --- a/precommit/core.d/patchfiles.sh +++ b/precommit/core.d/patchfiles.sh @@ -97,7 +97,6 @@ function locate_patch # it's a declarely provided file if [[ -f ${PATCH_OR_ISSUE} ]]; then patchfile="${PATCH_OR_ISSUE}" - PATCH_SYSTEM=generic else # run through the bug systems. maybe they know? for bugsys in ${BUGSYSTEMS}; do @@ -105,7 +104,6 @@ function locate_patch "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" if [[ $? == 0 ]]; then gotit=true - PATCH_SYSTEM=${bugsys} fi fi done @@ -117,12 +115,9 @@ function locate_patch yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}." cleanup_and_exit 1 fi - PATCH_SYSTEM=generic fi fi - yetus_debug "Determined patch system to be ${PATCH_SYSTEM}" - if [[ ! -f "${PATCH_DIR}/patch" && -f "${patchfile}" ]]; then cp "${patchfile}" "${PATCH_DIR}/patch" http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/test-patch-docker/Dockerfile ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile deleted file mode 100644 index ee92c33..0000000 --- a/precommit/test-patch-docker/Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:trusty - -WORKDIR /root - -###### -# Install common dependencies from packages -###### -RUN apt-get update && apt-get install --no-install-recommends -y \ - git curl ant make maven \ - cmake gcc g++ protobuf-compiler \ - build-essential libtool \ - zlib1g-dev pkg-config libssl-dev \ - snappy libsnappy-dev \ - bzip2 libbz2-dev \ - libjansson-dev \ - fuse libfuse-dev \ - libcurl4-openssl-dev \ - python python2.7 pylint \ - ruby \ - openjdk-7-jdk \ - libperl-critic-perl - -# Fixing the Apache commons / Maven dependency problem under Ubuntu: -# See http://wiki.apache.org/commons/VfsProblems -RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . - -####### -# Oracle Java -####### - -RUN apt-get install -y software-properties-common -RUN add-apt-repository -y ppa:webupd8team/java -RUN apt-get update - - -# Auto-accept the Oracle JDK license -RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get install -y oracle-java7-installer - -# Auto-accept the Oracle JDK license -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get install -y oracle-java8-installer - -###### -# Install findbugs -###### -RUN mkdir -p /opt/findbugs && \ - curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ - -o /opt/findbugs.tar.gz && \ - tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs -ENV FINDBUGS_HOME /opt/findbugs - -#### -# Install shellcheck -#### -RUN apt-get install -y cabal-install -RUN cabal update && cabal install shellcheck --global - -#### -# Install rubocop -### -RUN gem install rubocop --no-ri --no-rdoc - -#### -# Install ruby-lint -### -RUN gem install ruby-lint --no-ri --no-rdoc - -#### -# Install bats -#### -RUN add-apt-repository -y ppa:duggan/bats -RUN apt-get update -RUN apt-get install -y bats - -### -# Set the locale -### -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/test-patch-docker/Dockerfile-endstub ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile-endstub b/precommit/test-patch-docker/Dockerfile-endstub index ce76491..aa0463e 100644 --- a/precommit/test-patch-docker/Dockerfile-endstub +++ b/precommit/test-patch-docker/Dockerfile-endstub @@ -15,7 +15,5 @@ # limitations under the License. ADD launch-test-patch.sh /testptch/launch-test-patch.sh -RUN mkdir /testptch/extras -RUN chmod a+rwx /testptch/extras RUN chmod a+rx /testptch/launch-test-patch.sh CMD /testptch/launch-test-patch.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/test-patch-docker/Dockerfile-startstub ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/Dockerfile-startstub b/precommit/test-patch-docker/Dockerfile-startstub new file mode 100644 index 0000000..d191285 --- /dev/null +++ b/precommit/test-patch-docker/Dockerfile-startstub @@ -0,0 +1,91 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:trusty + +WORKDIR /root + +###### +# Install common dependencies from packages +###### +RUN apt-get update && apt-get install --no-install-recommends -y \ + git curl ant make maven \ + cmake gcc g++ protobuf-compiler \ + build-essential libtool \ + zlib1g-dev pkg-config libssl-dev \ + snappy libsnappy-dev \ + bzip2 libbz2-dev \ + libjansson-dev \ + fuse libfuse-dev \ + libcurl4-openssl-dev \ + python python2.7 pylint \ + ruby \ + openjdk-7-jdk \ + libperl-critic-perl + +# Fixing the Apache commons / Maven dependency problem under Ubuntu: +# See http://wiki.apache.org/commons/VfsProblems +RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . + +####### +# Oracle Java +####### + +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:webupd8team/java +RUN apt-get update + + +# Auto-accept the Oracle JDK license +RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +RUN apt-get install -y oracle-java7-installer + +# Auto-accept the Oracle JDK license +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +RUN apt-get install -y oracle-java8-installer + +###### +# Install findbugs +###### +RUN mkdir -p /opt/findbugs && \ + curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ + -o /opt/findbugs.tar.gz && \ + tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs +ENV FINDBUGS_HOME /opt/findbugs + +#### +# Install shellcheck +#### +RUN apt-get install -y cabal-install +RUN cabal update && cabal install shellcheck --global + +#### +# Install rubocop +### +RUN gem install rubocop + +#### +# Install ruby-lint +### +RUN gem install ruby-lint + +#### +# Install bats +#### +RUN add-apt-repository -y ppa:duggan/bats +RUN apt-get update +RUN apt-get install -y bats http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/test-patch-docker/launch-test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/launch-test-patch.sh b/precommit/test-patch-docker/launch-test-patch.sh index 67c7dfd..ac976ad 100755 --- a/precommit/test-patch-docker/launch-test-patch.sh +++ b/precommit/test-patch-docker/launch-test-patch.sh @@ -38,16 +38,6 @@ TESTPATCHMODE=${TESTPATCHMODE/--docker } cd "${BASEDIR}" PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P) -# if patch system is generic, then it's either a local -# patch file or was in some other way not pulled from a bug -# system. So we need to rescue it and then tell -# test-patch where to find it. -if [[ "${PATCH_SYSTEM}" = generic ]]; then - cp -p "${PATCH_DIR}/patch" /testptch/extras/patch - patchfile="/testptch/extras/patch" -fi - - cd "${PATCH_DIR}/precommit/" #shellcheck disable=SC2086 "${PATCH_DIR}/precommit/test-patch.sh" \ @@ -56,5 +46,5 @@ cd "${PATCH_DIR}/precommit/" --basedir="${BASEDIR}" \ --patch-dir="${PATCH_DIR}" \ --java-home="${JAVA_HOME}" \ - --user-plugins="${PATCH_DIR}/precommit/user-plugins" \ - ${patchfile} + --personality="${PATCH_DIR}/precommit/personality/provided.sh" \ + --user-plugins="${PATCH_DIR}/precommit/user-plugins" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/test-patch-docker/test-patch-docker.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch-docker/test-patch-docker.sh b/precommit/test-patch-docker/test-patch-docker.sh new file mode 100755 index 0000000..ca910ff --- /dev/null +++ b/precommit/test-patch-docker/test-patch-docker.sh @@ -0,0 +1,392 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DID=${RANDOM} + +## @description Print a message to stderr if --debug is turned on +## @audience private +## @stability stable +## @replaceable no +## @param string +function yetus_debug +{ + if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then + echo "[$(date) DEBUG]: $*" 1>&2 + fi +} + +## @description Run docker with some arguments, and +## @description optionally send to debug +## @audience private +## @stability evolving +## @replaceable no +## @param args +function dockercmd +{ + yetus_debug "docker $*" + docker "$@" +} + +## @description Handle command line arguments +## @audience private +## @stability evolving +## @replaceable no +## @param args +function parse_args +{ + local i + + for i in "$@"; do + case ${i} in + --debug) + YETUS_SHELL_SCRIPT_DEBUG=true + ;; + --dockerversion=*) + DOCKER_VERSION=${i#*=} + ;; + --help|-help|-h|help|--h|--\?|-\?|\?) + yetus_usage + exit 0 + ;; + --java-home=*) + JAVA_HOME=${i#*=} + ;; + --patch-dir=*) + PATCH_DIR=${i#*=} + ;; + --project=*) + PROJECT_NAME=${i#*=} + ;; + *) + ;; + esac + done +} + +## @description Stop and delete all defunct containers +## @audience private +## @stability evolving +## @replaceable no +## @param args +function stop_exited_containers +{ + local line + local id + local value + local size + + echo "Docker containers in exit state:" + + dockercmd ps -a | grep Exited + + # stop *all* containers that are in exit state for + # more than > 8 hours + while read -r line; do + id=$(echo "${line}" | cut -f1 -d' ') + value=$(echo "${line}" | cut -f2 -d' ') + size=$(echo "${line}" | cut -f3 -d' ') + + if [[ ${size} =~ day + || ${size} =~ week + || ${size} =~ month + || ${size} =~ year ]]; then + echo "Removing docker ${id}" + dockercmd rm "${id}" + fi + + if [[ ${size} =~ hours + && ${value} -gt 8 ]]; then + echo "Removing docker ${id}" + dockercmd rm "${id}" + fi + done < <( + dockercmd ps -a \ + | grep Exited \ + | sed -e 's,ago,,g' \ + | awk '{print $1" "$(NF - 2)" "$(NF - 1)}') +} + +## @description Remove all containers that are not +## @description are not running + older than 1 day +## @audience private +## @stability evolving +## @replaceable no +## @param args +function rm_old_containers +{ + declare line + declare id + declare value + declare size + declare running + + while read -r line; do + id=$(echo "${line}" | cut -f1 -d, ) + running=$(echo "${line}" | cut -f2 -d, ) + stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. ) + + if [[ ${running} = true ]]; then + yetus_debug "${id} is still running, skipping." + continue + fi + + # believe it or not, date is not even close to standardized... + if [[ $(uname -s) == Linux ]]; then + + # GNU date + stoptime=$(date -d "${stoptime}" "+%s") + else + + # BSD date + stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s") + fi + + curtime=$(date "+%s") + ((difftime = curtime - stoptime)) + if [[ ${difftime} -gt 86400 ]]; then + echo "Removing docker ${id}" + dockercmd rm "${id}" + fi + done < <( + # see https://github.com/koalaman/shellcheck/issues/375 + # shellcheck disable=SC2046 + dockercmd inspect \ + -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \ + $(dockercmd ps -qa) 2>/dev/null) +} + +## @description Remove untagged/unused images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function remove_untagged_images +{ + # this way is a bit more compatible with older docker versions + dockercmd images | tail -n +2 | awk '$1 == "<none>" {print $3}' | \ + xargs --no-run-if-empty docker rmi +} + +## @description Remove defunct tagged images +## @audience private +## @stability evolving +## @replaceable no +## @param args +function remove_old_tagged_images +{ + local line + local id + local created + + while read -r line; do + id=$(echo "${line}" | awk '{print $1}') + created=$(echo "${line}" | awk '{print $5}') + + if [[ ${created} =~ week + || ${created} =~ month + || ${created} =~ year ]]; then + echo "Removing docker image ${id}" + dockercmd rmi "${id}" + fi + + if [[ ${id} =~ test-patch-base-${PROJECT_NAME}-date ]]; then + if [[ ${created} =~ day + || ${created} =~ hours ]]; then + echo "Removing docker image ${id}" + dockercmd rmi "${id}" + fi + fi + done < <(dockercmd images) + +} + +## @description Performance docker maintenance on Jenkins +## @audience private +## @stability evolving +## @replaceable no +## @param args +function cleanup_apache_jenkins_docker +{ + echo "==========================" + echo "Docker Images:" + dockercmd images + echo "==========================" + echo "Docker Containers:" + dockercmd ps -a + echo "==========================" + + stop_exited_containers + + rm_old_containers + + remove_untagged_images + + remove_old_tagged_images +} + +## @description Clean up our old images used for patch testing +## @audience private +## @stability evolving +## @replaceable no +## @param args +function cleanup_test_patch_images +{ + local images + local imagecount + local rmimage + local rmi + + # we always want to leave at least one of our images + # so that the whole thing doesn't have to be rebuilt. + # This also let's us purge any old images so that + # we can get fresh stuff sometimes + images=$(dockercmd images | grep --color=none "test-patch-tp-${PROJECT_NAME}" | awk '{print $1}') 2>&1 + + # shellcheck disable=SC2086 + imagecount=$(echo ${images} | tr ' ' '\n' | wc -l) + ((imagecount = imagecount - 1 )) + + # shellcheck disable=SC2086 + rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount}) + for rmi in ${rmimage} + do + echo "Removing image ${rmi}" + dockercmd rmi "${rmi}" + done +} + +## @description Perform pre-run maintenance to free up +## @description resources. With --jenkins, it is a lot +## @description more destructive. +## @audience private +## @stability evolving +## @replaceable no +## @param args +function cleanup +{ + if [[ ${TESTPATCHMODE} =~ jenkins ]]; then + cleanup_apache_jenkins_docker + fi + + cleanup_test_patch_images +} + +## @description Deterine the user name and user id of the user +## @description that the docker container should use +## @audience private +## @stability evolving +## @replaceable no +## @param args +function determine_user +{ + # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some + # ... person ... sets it to an account that doesn't actually exist. + # so instead, we need to try and override it with something that's + # probably close to reality. + if [[ ${TESTPATCHMODE} =~ jenkins ]]; then + USER=$(id | cut -f2 -d\( | cut -f1 -d\)) + fi + + if [[ "$(uname -s)" == "Linux" ]]; then + USER_NAME=${SUDO_USER:=$USER} + USER_ID=$(id -u "${USER_NAME}") + GROUP_ID=$(id -g "${USER_NAME}") + else # boot2docker uid and gid + USER_NAME=${USER} + USER_ID=1000 + GROUP_ID=50 + fi +} + +## @description Determine the revision of a dockerfile +## @audience private +## @stability evolving +## @replaceable no +## @param args +function getdockerfilerev +{ + grep 'TEST_PATCH_PRIVATE: gitrev=' \ + "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \ + | cut -f2 -d= +} + +## @description Start a test patch docker container +## @audience private +## @stability evolving +## @replaceable no +## @param args +function run_image +{ + local dockerfilerev + local baseimagename + + dockerfilerev=$(getdockerfilerev) + + baseimagename="test-patch-base-${PROJECT_NAME}-${dockerfilerev}" + + # make a base image, if it isn't available + dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker" + + # using the base image, make one that is patch specific + dockercmd build -t "test-patch-tp-${PROJECT_NAME}-${DID}" - <<PatchSpecificDocker +FROM ${baseimagename} +RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} +RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} +RUN chown -R ${USER_NAME} /home/${USER_NAME} +ENV HOME /home/${USER_NAME} +USER ${USER_NAME} +PatchSpecificDocker + + if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then + extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt") + else + extraargs="" + fi + + if [[ ${PATCH_DIR} =~ ^/ ]]; then + dockercmd run --rm=true -i \ + ${extraargs} \ + -v "${PWD}:/testptch/${PROJECT_NAME}" \ + -v "${PATCH_DIR}:/testptch/patchprocess" \ + -u "${USER_NAME}" \ + -w "/testptch/${PROJECT_NAME}" \ + --env=BASEDIR="/testptch/${PROJECT_NAME}" \ + --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ + --env=JAVA_HOME="${JAVA_HOME}" \ + --env=PATCH_DIR=/testptch/patchprocess \ + --env=PROJECT_NAME="${PROJECT_NAME}" \ + --env=TESTPATCHMODE="${TESTPATCHMODE}" \ + "test-patch-tp-${PROJECT_NAME}-${DID}" + else + dockercmd run --rm=true -i \ + ${extraargs} \ + -v "${PWD}:/testptch/${PROJECT_NAME}" \ + -u "${USER_NAME}" \ + -w "/testptch/${PROJECT_NAME}" \ + --env=BASEDIR="/testptch/${PROJECT_NAME}" \ + --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \ + --env=JAVA_HOME="${JAVA_HOME}" \ + --env=PATCH_DIR="${PATCH_DIR}" \ + --env=PROJECT_NAME="${PROJECT_NAME}" \ + --env=TESTPATCHMODE="${TESTPATCHMODE}" \ + "test-patch-tp-${PROJECT_NAME}-${DID}" + fi +} + +parse_args "$@" +cleanup +determine_user +run_image http://git-wip-us.apache.org/repos/asf/yetus/blob/23a1c9af/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index 7420a05..1847848 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -56,7 +56,7 @@ function setup_defaults { common_defaults - DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile" + DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub" HOW_TO_CONTRIBUTE="https://yetus.apache.org/documentation/latest/precommit-patchnames"; INSTANCE=${RANDOM} RELOCATE_PATCH_DIR=false @@ -677,7 +677,6 @@ function yetus_usage echo "--debug If set, then output some extra stuff to stderr" echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" echo "--docker Spawn a docker container" - echo "--dockercmd=<file> Command to use as docker executable (default: docker from path)" echo "--dockerfile=<file> Dockerfile fragment to use as the base" echo "--java-home=<path> Set JAVA_HOME (In Docker mode, this should be local to the image)" echo "--linecomments=<bug> Only write line comments to this comma delimited list (defaults to bugcomments)" @@ -763,9 +762,6 @@ function parse_args --docker) DOCKERSUPPORT=true ;; - --dockercmd=*) - DOCKERCMD=${i#*=} - ;; --dockerfile=*) DOCKERFILE=${i#*=} ;; @@ -916,12 +912,6 @@ function parse_args GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt" GITUNIDIFFLINES="${PATCH_DIR}/gitdiffunilines.txt" - if [[ -n "${REEXECPERSONALITY}" - && -f "${PATCH_DIR}/precommit/personality/provided.sh" ]]; then - PERSONALITY="${PATCH_DIR}/precommit/personality/provided.sh" - fi - - DOCKERFILE=$(yetus_abs "${DOCKERFILE}") } ## @description Locate the build file for a given directory @@ -1373,11 +1363,9 @@ function apply_patch_file ## @replaceable no function copytpbits { - declare dockerdir - declare dockfile - declare person - declare lines - + local dockerdir + local dockfile + local person # we need to copy/consolidate all the bits that might have changed # that are considered part of test-patch. This *might* break # things that do off-path includes, but there isn't much we can @@ -1426,18 +1414,12 @@ function copytpbits gitfilerev="date${gitfilerev}" fi ( - echo "### YETUS_PRIVATE: dockerfile=${DOCKERFILE}" - echo "### YETUS_PRIVATE: gitrev=${gitfilerev}" - lines=$(${GREP} -n 'YETUS CUT HERE' ${DOCKERFILE} | cut -f1 -d:) - if [[ -z "${lines}" ]]; then - cat "${DOCKERFILE}" - else - head -n "${lines}" "${DOCKERFILE}" - fi + echo "### TEST_PATCH_PRIVATE: dockerfile=${DOCKERFILE}" + echo "### TEST_PATCH_PRIVATE: gitrev=${gitfilerev}" + cat "${DOCKERFILE}" # make sure we put some space between, just in case last # line isn't an empty line or whatever printf "\n\n" - echo "### YETUS_PRIVATE: start test-patch-bootstrap" cat "${BINDIR}/test-patch-docker/Dockerfile-endstub" printf "\n\n" @@ -1448,34 +1430,6 @@ function copytpbits popd >/dev/null } -## @description Verify docker exists -## @audience private -## @stability evolving -## @replaceable no -## @returns 1 if docker not found -## @returns 0 if docker is found -function dockerverify -{ - declare pathdocker - - if [[ -z "${DOCKERCMD}" ]]; then - pathdocker=$(which docker 2>/dev/null) - - if [[ ! -f "${pathdocker}" ]]; then - yetus_error "Docker cannot be found." - return 1 - fi - DOCKERCMD="${pathdocker}" - fi - - if [[ ! -x "${DOCKERCMD}" ]];then - yetus_error "Docker command ${DOCKERCMD} is not executable." - return 1 - fi - return 0 -} - - ## @description If this patches actually patches test-patch.sh, then ## @description run with the patched version for the test. ## @audience private @@ -1496,10 +1450,6 @@ function check_reexec return fi - # determine if the patch hits - # any test-patch sensitive bits - # if so, we need to copy the universe - # after patching it (copy=true) for testdir in "${BINDIR}" \ "${PERSONALITY}" \ "${USER_PLUGIN_DIR}" \ @@ -1511,14 +1461,6 @@ function check_reexec fi done - if [[ "${DOCKERSUPPORT}" = true ]]; then - dockerverify - if [[ $? == 1 ]]; then - yetus_error "Docker not found or not executable. Disabling Docker mode." - DOCKERSUPPORT=false - fi - fi - if [[ ${copy} == true ]]; then big_console_header "precommit patch detected" @@ -1547,8 +1489,8 @@ function check_reexec fi if [[ ${DOCKERSUPPORT} == true - && ${copy} == false ]]; then - big_console_header "Re-execing under Docker" + && ${copy} == false ]]; then + big_console_header "Re-execing under Docker" fi # copy our universe @@ -1571,27 +1513,29 @@ function check_reexec if [[ -n "${BUILD_URL}" ]]; then TESTPATCHMODE="--build-url=${BUILD_URL} ${TESTPATCHMODE}" fi - - if [[ -f "${PERSONALITY}" ]]; then - TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}" - fi - TESTPATCHMODE="--tpglobaltimer=${GLOBALTIMER} ${TESTPATCHMODE}" TESTPATCHMODE="--tpreexectimer=${TIMER} ${TESTPATCHMODE}" TESTPATCHMODE="--tpinstance=${INSTANCE} ${TESTPATCHMODE}" + TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}" TESTPATCHMODE="--plugins=${ENABLED_PLUGINS// /,} ${TESTPATCHMODE}" TESTPATCHMODE=" ${TESTPATCHMODE}" export TESTPATCHMODE patchdir=$(relative_dir "${PATCH_DIR}") - if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then + if [[ ${TP_SHELL_SCRIPT_DEBUG} = true ]]; then debugflag="--debug" fi cd "${BASEDIR}" #shellcheck disable=SC2093 - docker_handler + exec bash "${PATCH_DIR}/precommit/test-patch-docker/test-patch-docker.sh" \ + ${debugflag} \ + --dockerversion="${dockerversion}" \ + --java-home="${JAVA_HOME}" \ + --patch-dir="${patchdir}" \ + --project="${PROJECT_NAME}" + else # if we aren't doing docker, then just call ourselves @@ -2620,7 +2564,7 @@ function prechecks declare plugin declare result=0 - for plugin in ${BUILDTOOL} ${NEEDEDTESTS} ${TESTFORMATS}; do + for plugin in ${BUILDTOOL} ${TESTTYPES} ${TESTFORMATS}; do verify_patchdir_still_exists if declare -f ${plugin}_precheck >/dev/null 2>&1; then
