YETUS-523: create a separate tool for docker cleanup Signed-off-by: Sean Busbey <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/10d4d13c Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/10d4d13c Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/10d4d13c Branch: refs/heads/master Commit: 10d4d13cc95a814eac97a976a8de525531ac986a Parents: eabc378 Author: Allen Wittenauer <[email protected]> Authored: Fri Jun 23 08:59:40 2017 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Fri Jun 30 11:34:27 2017 -0700 ---------------------------------------------------------------------- .../in-progress/precommit-docker-cleanup.md | 60 ++++++ build.sh | 7 +- precommit/core.d/docker.sh | 25 +-- precommit/docker-cleanup.sh | 185 +++++++++++++++++++ precommit/test-patch.sh | 2 +- 5 files changed, 265 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md b/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md new file mode 100644 index 0000000..9df7524 --- /dev/null +++ b/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md @@ -0,0 +1,60 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +docker-cleanup +============== + +`docker-cleanup` is a command to perform precommit's Docker cleanup functionality outside of patch and build testing. + +# Usage + +Executing `docker-cleanup` will peform the same Docker image as the `--robot` option. + +```bash +$ docker-cleanup + +... + Removing old images +... +Untagged: hadoop/createrelease:3.0.0-alpha3-SNAPSHOT_10600 +Deleted: sha256:1a96c79a0a9ab538c6c7765dc908eca3e689270b778d6ae2add558e89792a7d8 +... + Docker Container Maintenance +... + +``` + + +`docker-cleanup` also supports the `--sentinel` mode to kill and remove stale running containers: + +```bash +$ docker-cleanup --sentinel + +... + Removing old images +... +Untagged: hadoop/createrelease:3.0.0-alpha3-SNAPSHOT_10600 +Deleted: sha256:1a96c79a0a9ab538c6c7765dc908eca3e689270b778d6ae2add558e89792a7d8 +... + + Docker Container Maintenance +... +Attempting to remove docker container /big_kowalevski [5ffd22a56ebcfe38d72b9078e0e7133ab6dc054115a4804e504c910bdbdeea45] +... +``` http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/build.sh ---------------------------------------------------------------------- diff --git a/build.sh b/build.sh index d9fbe89..6783596 100755 --- a/build.sh +++ b/build.sh @@ -214,8 +214,11 @@ exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/release-doc-maker/releasedocmake EOF chmod +x "${bin_tarball}/bin/releasedocmaker" -for utility in shelldocs/shelldocs.py precommit/qbt.sh \ - precommit/smart-apply-patch.sh precommit/test-patch.sh +for utility in shelldocs/shelldocs.py \ + precommit/docker-cleanup.sh \ + precommit/qbt.sh \ + precommit/smart-apply-patch.sh \ + precommit/test-patch.sh do wrapper=${utility##*/} wrapper=${wrapper%.*} http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/core.d/docker.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh index 863b045..124aa0b 100755 --- a/precommit/core.d/docker.sh +++ b/precommit/core.d/docker.sh @@ -22,6 +22,7 @@ DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile" DOCKERFAIL="fallback,continue,fail" DOCKERSUPPORT=false DOCKER_ENABLE_PRIVILEGED=true +DOCKER_CLEANUP_CMD=false declare -a DOCKER_EXTRAARGS @@ -43,11 +44,15 @@ DOCKER_IMAGE_PURGE=604800 ## @replaceable no function docker_usage { - yetus_add_option "--docker" "Spawn a docker container" + if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then + yetus_add_option "--docker" "Spawn a docker container" + fi yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default: '${DOCKERCMD}')" - yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')" - yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})" - yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')" + if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then + yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')" + yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})" + yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')" + fi yetus_add_option "--dockerdelrep" "In Docker mode, only report image/container deletions, not act on them" } @@ -129,8 +134,7 @@ function docker_initialize DOCKERFAIL=${DOCKERFAIL//fail/3} DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/} - docker_exeverify - if [[ $? != 0 ]]; then + if ! docker_exeverify; then if [[ "${DOCKERFAIL}" =~ ^12 || "${DOCKERFAIL}" =~ ^2 ]]; then add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling docker." @@ -205,9 +209,7 @@ function docker_exeverify return 1 fi - ${DOCKERCMD} info >/dev/null 2>&1 - - if [[ $? != 0 ]]; then + if ! ${DOCKERCMD} info >/dev/null 2>&1; then yetus_error "Docker is not functioning properly. Daemon down/unreachable?" return 1 fi @@ -487,12 +489,13 @@ function docker_version { declare vertype=$1 declare val - + declare ret # new version command val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null) + ret=$? - if [[ $? != 0 ]];then + if [[ ${ret} != 0 ]];then # old version command val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ') fi http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/docker-cleanup.sh ---------------------------------------------------------------------- diff --git a/precommit/docker-cleanup.sh b/precommit/docker-cleanup.sh new file mode 100755 index 0000000..4d18d21 --- /dev/null +++ b/precommit/docker-cleanup.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no shelldocs required from this file +# SHELLDOC-IGNORE + +# Make sure that bash version meets the pre-requisite + +if [[ -z "${BASH_VERSINFO[0]}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + +this="${BASH_SOURCE-$0}" +BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +BINNAME=${this##*/} +BINNAME=${BINNAME%.sh} +#shellcheck disable=SC2034 +STARTINGDIR=$(pwd) +#shellcheck disable=SC2034 +USER_PARAMS=("$@") +#shellcheck disable=SC2034 +QATESTMODE=false + +# dummy functions +function add_vote_table +{ + true +} + +function add_footer_table +{ + true +} + +function bugsystem_finalreport +{ + true +} + +## @description import core library routines +## @audience private +## @stability evolving +function import_core +{ + declare filename + + for filename in "${BINDIR}/core.d"/*; do + # shellcheck disable=SC1091 + # shellcheck source=core.d/01-common.sh + . "${filename}" + done +} + +## @description import plugins then remove the stuff we don't need +## @audience public +## @stability stable +## @replaceable no +function import_and_clean +{ + importplugins + yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list" + unset BUILDTOOLS + unset TESTTYPES + unset TESTFORMATS + + #shellcheck disable=SC2034 + DOCKER_CLEANUP_CMD=true + #shellcheck disable=SC2034 + DOCKERSUPPORT=true + #shellcheck disable=SC2034 + ROBOT=true + #shellcheck disable=SC2034 + DOCKERFAIL="fail" +} + +## @description Clean the filesystem as appropriate and then exit +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function cleanup_and_exit +{ + local result=$1 + + if [[ ${PATCH_DIR} =~ ^/tmp/yetus + && -d ${PATCH_DIR} ]]; then + rm -rf "${PATCH_DIR}" + fi + + # shellcheck disable=SC2086 + exit ${result} +} + +## @description Setup the default global variables +## @audience public +## @stability stable +## @replaceable no +function setup_defaults +{ + common_defaults +} + +## @description Interpret the command line parameters +## @audience private +## @stability stable +## @replaceable no +## @param $@ +## @return May exit on failure +function parse_args +{ + common_args "$@" +} + +## @description Print the usage information +## @audience public +## @stability stable +## @replaceable no +function yetus_usage +{ + import_and_clean + + echo "${BINNAME} [OPTIONS]" + + yetus_add_option "--debug" "If set, then output some extra stuff to stderr" + yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)" + docker_usage + + yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" + yetus_reset_usage +} + +## @description Large display for the user console +## @audience public +## @stability stable +## @replaceable no +## @param string +## @return large chunk of text +function big_console_header +{ + local text="$*" + local spacing=$(( (75+${#text}) /2 )) + printf "\n\n" + echo "============================================================================" + echo "============================================================================" + printf "%*s\n" ${spacing} "${text}" + echo "============================================================================" + echo "============================================================================" + printf "\n\n" +} + +trap "cleanup_and_exit 1" HUP INT QUIT TERM + +import_core + +setup_defaults + +parse_args "$@" + +import_and_clean + +parse_args_plugins "$@" + +docker_initialize +plugins_initialize + +docker_cleanup +RESULT=$? + +cleanup_and_exit ${RESULT} http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index a632f5f..75dbbe4 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -2994,7 +2994,7 @@ function initialize BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}} - # we need to do this BEFORE plugins initalize + # we need to do this BEFORE plugins initialize # because they may change what they do based upon # docker support # note that docker support still isn't guaranteed
