Repository: yetus Updated Branches: refs/heads/master 6e95a5631 -> 37b482ed0
YETUS-186. non-existent dockerfile should fail or fallback Signed-off-by: Sean Busbey <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/37b482ed Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/37b482ed Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/37b482ed Branch: refs/heads/master Commit: 37b482ed0ca28fbc47503fb972950c08d6ec96cb Parents: 6e95a56 Author: Allen Wittenauer <[email protected]> Authored: Wed Nov 18 13:47:45 2015 -0800 Committer: Allen Wittenauer <[email protected]> Committed: Thu Nov 19 09:31:51 2015 -0800 ---------------------------------------------------------------------- .../in-progress/precommit-advanced.md | 12 ++++ precommit/core.d/docker.sh | 65 ++++++++++++++++++-- precommit/test-patch.sh | 28 ++++----- 3 files changed, 82 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/37b482ed/asf-site-src/source/documentation/in-progress/precommit-advanced.md ---------------------------------------------------------------------- diff --git a/asf-site-src/source/documentation/in-progress/precommit-advanced.md b/asf-site-src/source/documentation/in-progress/precommit-advanced.md index dda9546..e404988 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-advanced.md +++ b/asf-site-src/source/documentation/in-progress/precommit-advanced.md @@ -31,6 +31,18 @@ By default, test-patch runs in the same shell where it was launched. It can alt The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. test-patch will copy this file up until the text "YETUS CUT HERE" to a different directory and then append its necessary hooks to re-launch itself prior to executing docker. +If a custom Dockerfile cannot be used or the docker executable does not work, test-patch will attempt to recover by switching to its bundled Dockerfile or disabling docker support and running locally. This behavior can be changed with the `--dockeronfail` option. It takes a list of comma-delimited settings: + + * fallback - Use the bundled Dockerfile + * continue - Turn off docker support + * fail - fail the test + +The 'fail' setting is always the last option that test-patch will use and may be omitted unless it is the only option. + +For example, `--dockeronfail=continue` means if the Dockerfile can't be found, just turn off Docker support and continue running. `--dockeronfail=fallback` will switch to the bundled Dockerfile and then fail the build if docker fails to work. `--dockeronfail=fail` means to just fail the build and do not try any other mechanisms of recovery. The default is 'fallback,continue,fail' which will allow test-patch to try to continue executing as much as it possibily can. + +Be aware that if the Dockerfile is found and the docker command works, test-patch will always fail the build if the Dockerfile itself fails the build. It will not attempt to continue in the non-Docker mode. + NOTE: If you are using Boot2Docker, you must use directories under /Users (OSX) or C:\Users (Windows) as the base and patchprocess directories (specified by the --basedir and --patch-dir options respectively), because automatically mountable directories are limited to them. See [the Docker documentation](https://docs.docker.com/userguide/dockervolumes/#mount-a-host-directory-as-a-data-volume). Dockerfile images will be named with a test-patch prefix and suffix with either a date or a git commit hash. By using this information, test-patch will automatically manage broken/stale container images that are hanging around if it is run in --jenkins mode. In this way, if Docker fails to build the image, the disk space should eventually be cleaned and returned back to the system. http://git-wip-us.apache.org/repos/asf/yetus/blob/37b482ed/precommit/core.d/docker.sh ---------------------------------------------------------------------- diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh index f121274..ceb3e19 100755 --- a/precommit/core.d/docker.sh +++ b/precommit/core.d/docker.sh @@ -16,25 +16,78 @@ DOCKER_ID=${RANDOM} DOCKER_DESTRUCTIVE=true +DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile" +DOCKERFAIL="fallback,continue,fail" -## @description Put docker stats in various tables -## @stability stable +## @description Docker initialization pre- and post- re-exec +## @stability stable ## @audience private -## @replaceable yes -function docker_finish_stats +## @replaceable no +function docker_initialize { if [[ ${DOCKERMODE} == true ]]; then # DOCKER_VERSION is set by our creator. add_footer_table "Docker" "${DOCKER_VERSION}" fi + + + # turn DOCKERFAIL into a string composed of numbers + # to ease interpretation: 123, 213, 321, ... whatever + # some of these combos are non-sensical but that's ok. + # we'll treat non-sense as effectively errors. + DOCKERFAIL=${DOCKERFAIL//,/ } + DOCKERFAIL=${DOCKERFAIL//fallback/1} + DOCKERFAIL=${DOCKERFAIL//continue/2} + DOCKERFAIL=${DOCKERFAIL//fail/3} + DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/} + + if [[ "${DOCKERSUPPORT}" != true ]]; then + return + fi + + if [[ -n "${DOCKERFILE}" ]]; then + pushd "${STARTINGDIR}" >/dev/null + if [[ -f ${DOCKERFILE} ]]; then + DOCKERFILE=$(yetus_abs "${DOCKERFILE}") + else + if [[ "${DOCKERFAIL}" =~ ^1 ]]; then + yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, falling back to built-in." + add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, falling back to built-in." + DOCKERFILE=${DOCKERFILE_DEFAULT} + elif [[ "${DOCKERFAIL}" =~ ^2 ]]; then + yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, disabling docker." + add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, disabling docker." + DOCKERSUPPORT=false + else + yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found." + add_vote_table -1 docker "Dockerfile '${DOCKERFILE}' not found." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + fi + popd >/dev/null + fi + + dockerverify + if [[ $? != 0 ]]; then + if [[ "${DOCKERFAIL}" =~ ^12 + || "${DOCKERFAIL}" =~ ^2 ]]; then + add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling docker." + DOCKERSUPPORT=false + else + add_vote_table -1 docker "Docker command '${DOCKERCMD}' not found/broken." + bugsystem_finalreport 1 + cleanup_and_exit 1 + fi + fi } ## @description Verify docker exists ## @audience private ## @stability evolving ## @replaceable no -## @returns 1 if docker not found -## @returns 0 if docker is found +## @returns 1 if docker is broken +## @returns 0 if docker is working function dockerverify { declare pathdocker http://git-wip-us.apache.org/repos/asf/yetus/blob/37b482ed/precommit/test-patch.sh ---------------------------------------------------------------------- diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh index 16b8b30..7ee8fc8 100755 --- a/precommit/test-patch.sh +++ b/precommit/test-patch.sh @@ -56,7 +56,6 @@ function setup_defaults { common_defaults - DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile" HOW_TO_CONTRIBUTE="https://yetus.apache.org/documentation/latest/precommit-patchnames"; INSTANCE=${RANDOM} RELOCATE_PATCH_DIR=false @@ -646,8 +645,6 @@ function relative_dir ## @replaceable no function yetus_usage { - local -r up=$(echo "${PROJECT_NAME}" | tr '[:lower:]' '[:upper:]') - echo "Usage: test-patch.sh [options] patch" echo echo "Where:" @@ -667,6 +664,7 @@ function yetus_usage echo "--docker Spawn a docker container" echo "--dockercmd=<file> Command to use as docker executable (default: docker from path)" echo "--dockerfile=<file> Dockerfile fragment to use as the base" + echo "--dockeronfail=<list> If Docker fails, determine fallback method order (default: ${DOCKERFAIL})" echo "--java-home=<path> Set JAVA_HOME (In Docker mode, this should be local to the image)" echo "--linecomments=<bug> Only write line comments to this comma delimited list (defaults to bugcomments)" echo "--list-plugins List all installed plug-ins and then exit" @@ -761,6 +759,9 @@ function parse_args --dockermode) DOCKERMODE=true ;; + --dockeronfail=*) + DOCKERFAIL=${i#*=} + ;; --java-home=*) JAVA_HOME=${i#*=} ;; @@ -907,8 +908,6 @@ function parse_args REEXECPERSONALITY="${PERSONALITY}" PERSONALITY="${PATCH_DIR}/precommit/personality/provided.sh" fi - - DOCKERFILE=$(yetus_abs "${DOCKERFILE}") } ## @description Locate the build file for a given directory @@ -1468,14 +1467,6 @@ function check_reexec fi done - if [[ "${DOCKERSUPPORT}" = true ]]; then - dockerverify - if [[ $? == 1 ]]; then - yetus_error "Docker not found or not executable. Disabling Docker mode." - DOCKERSUPPORT=false - fi - fi - if [[ ${copy} == true ]]; then big_console_header "precommit patch detected" @@ -2526,12 +2517,15 @@ function initialize BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}} - plugins_initialize - - if declare -f docker_finish_stats >/dev/null; then - docker_finish_stats + # we need to do this BEFORE plugins initalize + # because they may change what they do based upon + # docker support + if declare -f docker_initialize >/dev/null; then + docker_initialize fi + plugins_initialize + locate_patch # from here on out, we'll be in ${BASEDIR} for cwd
