This is an automated email from the ASF dual-hosted git repository. aw pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 8f31a4c8c88a314e7c76fa212a9abb75a1c98c94 Author: Allen Wittenauer <[email protected]> AuthorDate: Sun Jan 6 10:42:49 2019 -0800 YETUS-764. test-patch running in a pipeline isn't getting killed Signed-off-by: Sean Busbey <[email protected]> --- Jenkinsfile | 22 ++++++++++++++++-- .../documentation/in-progress/precommit-robots.md | 27 ++++++++++++++++++++-- precommit/src/main/shell/core.d/docker.sh | 12 ++++++---- precommit/src/main/shell/test-patch.sh | 8 +++++++ 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 78a623a..aee1009 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -254,9 +254,27 @@ ${FILE,path="out/brief.txt"} } } - // Jenkins pipeline jobs fill slaves on PRs without this :( + // Jenkins has issues in some configurations sending + // signals to child processes. Additionally, Github Branch + // Source plug-in will quickly fill Jenkins build hosts + // with PR directories. This cleanup stanze kills + // any left over processes/containers and frees disk space + // on exit cleanup() { - deleteDir() + script { + sh ''' + if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then + echo "test-patch process appears to still be running: killing" + kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true + sleep 10 + fi + if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then + echo "test-patch container appears to still be running: killing" + docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true + fi + ''' + deleteDir() + } } } } diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.md b/asf-site-src/source/documentation/in-progress/precommit-robots.md index faa487c..9c226ac 100644 --- a/asf-site-src/source/documentation/in-progress/precommit-robots.md +++ b/asf-site-src/source/documentation/in-progress/precommit-robots.md @@ -113,12 +113,35 @@ pipeline { JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64' } - ... - } ``` + Experience has shown that certain Jenkins + Java + OS combinations have problems sending signals to child processes. In the case of Apache Yetus, this may result in aborted or workflows that timeout not being properly killed. `test-patch` will write two files in the patch directory that may be helpful to combat this situation if it applies to your particular configuration. `pidfile.txt` contains the master `test-patch` process id and `cidfile.txt` contains the docker container id. T [...] + + ```groovy + post { + cleanup() { + script { + sh ''' + if [ -f "${env.PATCH_DIR}/pidfile.txt" ]; then + kill `cat "${env.PATCH_DIR}/pidfile.txt"` || true + sleep 5 + fi + if [ -f "${env.PATCH_DIR}/cidfile.txt" ]; then + docker kill `cat "${env.PATCH_DIR}/cidfile.txt"` || true + sleep 5 + fi + ''' + ... + deletedir() + } + } + } + ``` + + + See also * See also the source tree's `Jenkinsfile` for some tips and tricks. * [precommit-admin](precommit-admin), for special utilities built for Jenkins. diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh index d419bc0..d298d19 100755 --- a/precommit/src/main/shell/core.d/docker.sh +++ b/precommit/src/main/shell/core.d/docker.sh @@ -717,7 +717,7 @@ function docker_run_image # make the kernel prefer to kill us if we run out of RAM DOCKER_EXTRAARGS+=("--oom-score-adj" "500") - DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile") + DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile.txt") if [[ "${DOCKER_IN_DOCKER}" == true ]]; then if [[ -e "${DOCKER_SOCKET}" ]]; then @@ -735,8 +735,7 @@ function docker_run_image DOCKER_EXTRAARGS+=(--name "${containername}") - trap 'docker_signal_handler' SIGTERM - trap 'docker_signal_handler' SIGINT + trap 'docker_signal_handler' SIGTERM SIGINT SIGHUP if [[ ${PATCH_DIR} =~ ^/ ]]; then dockercmd run --rm=true -i \ @@ -758,6 +757,7 @@ function docker_run_image printf '\n\n' echo "Cleaning up docker image used for testing." dockercmd rmi "${patchimagename}" > /dev/null + rm "${PATCH_DIR}/cidfile.txt" cleanup_and_exit ${retval} } @@ -769,10 +769,12 @@ function docker_signal_handler { declare cid - cid=$(cat "${PATCH_DIR}/cidfile") + cid=$(cat "${PATCH_DIR}/cidfile.txt") yetus_error "ERROR: Caught signal. Killing docker container:" - dockercmd kill "${cid}" + echo "ERROR: Caught signal. Killing docker container: ${cid}" > "${PATCH_DIR}/signal.log" + dockercmd kill "${cid}" | tee -a "${PATCH_DIR}/signal.log" + rm "${PATCH_DIR}/cidfile.txt" yetus_error "ERROR: Exiting." cleanup_and_exit 143 # 128 + 15 -- SIGTERM } diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh index 4ea5cb1..b361289 100755 --- a/precommit/src/main/shell/test-patch.sh +++ b/precommit/src/main/shell/test-patch.sh @@ -2470,6 +2470,10 @@ function cleanup_and_exit big_console_header "Finished build." fi + if [[ "${DOCKERMODE}" != true ]]; then + rm "${PATCH_DIR}/pidfile.txt" + fi + # shellcheck disable=SC2086 exit ${result} } @@ -3210,6 +3214,10 @@ function initialize docker_initialize fi + if [[ "${DOCKERMODE}" != true ]]; then + echo "$$" > "${PATCH_DIR}/pidfile.txt" + fi + plugins_initialize if [[ ${RESULT} != 0 ]]; then cleanup_and_exit 1
