IMPALA-5031: Jenkins jobs should fail when UBSAN fails Before this patch, Jenkins builds do not fail or warn if UBSAN produces an error. This patch introduces two new failure modes, "error" and "death".
In "error", a junit error is generated, which causes a build to be labeled "unstable". In "death", a forked process constantly monitors the test logs and kills the test run if a UBSAN error is found. Change-Id: I783243458ac2765a97a1dd7dd40d458cc2e1d80b Reviewed-on: http://gerrit.cloudera.org:8080/11813 Reviewed-by: Jim Apple <jbapple-imp...@apache.org> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/62db9216 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/62db9216 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/62db9216 Branch: refs/heads/master Commit: 62db9216c6ca6f7a9b4c1048fe3fb5896db8add7 Parents: 58a469c Author: Jim Apple <jbapple-imp...@apache.org> Authored: Sat Oct 27 21:47:54 2018 -0700 Committer: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Committed: Sun Nov 11 05:02:05 2018 +0000 ---------------------------------------------------------------------- bin/jenkins/all-tests.sh | 34 ++++++++++++++++++++++++++++++++++ bin/jenkins/finalize.sh | 9 +++++++++ 2 files changed, 43 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/62db9216/bin/jenkins/all-tests.sh ---------------------------------------------------------------------- diff --git a/bin/jenkins/all-tests.sh b/bin/jenkins/all-tests.sh index d29f383..b531804 100644 --- a/bin/jenkins/all-tests.sh +++ b/bin/jenkins/all-tests.sh @@ -26,6 +26,40 @@ cd "${IMPALA_HOME}" export IMPALA_MAVEN_OPTIONS="-U" +# When UBSAN_FAIL is "death", the logs are monitored for UBSAN errors. Any errors will +# then cause this script to exit. +# +# When UBSAN_FAIL is "error", monitoring is delayed until tests have finished running. +# +# Any other value ignores UBSAN errors. +: ${UBSAN_FAIL:=error} +export UBSAN_FAIL + +if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \ + && [ "${UBSAN_FAIL}" = "death" ] +then + export PID_TO_KILL="$(echo $$)" + mkdir -p "${IMPALA_HOME}/logs" + + function killer { + while ! grep -rI ": runtime error: " "${IMPALA_HOME}/logs" + do + sleep 1 + if ! test -e "/proc/$PID_TO_KILL" + then + return + fi + done + >&2 echo "Killing process $PID_TO_KILL because it invoked undefined behavior" + kill -9 $PID_TO_KILL + } + + killer & + export KILLER_PID="$(echo $!)" + disown + trap "kill -i $KILLER_PID" EXIT +fi + source bin/bootstrap_development.sh RET_CODE=0 http://git-wip-us.apache.org/repos/asf/impala/blob/62db9216/bin/jenkins/finalize.sh ---------------------------------------------------------------------- diff --git a/bin/jenkins/finalize.sh b/bin/jenkins/finalize.sh index aaebb83..b7adfb4 100755 --- a/bin/jenkins/finalize.sh +++ b/bin/jenkins/finalize.sh @@ -21,6 +21,15 @@ set -euo pipefail trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \ + && [ "${UBSAN_FAIL}" = "error" ] \ + && { grep -rI ": runtime error: " "${IMPALA_HOME}/logs" 2>&1 | sort | uniq \ + | tee logs/ubsan.txt ; } +then + "${IMPALA_HOME}"/bin/generate_junitxml.py --step UBSAN \ + --stderr "${IMPALA_HOME}"/logs/ubsan.txt --error "Undefined C++ behavior" +fi + rm -rf "${IMPALA_HOME}"/logs_system mkdir -p "${IMPALA_HOME}"/logs_system dmesg > "${IMPALA_HOME}"/logs_system/dmesg
