IMPALA-5031: Jenkins jobs should fail when UBSAN fails Before this patch, Jenkins builds do not fail or warn if UBSAN produces an error. This patch introduces two new failure modes, "error" and "death".
In "error", a junit error is generated, which causes a build to be labeled "unstable". In "death", a forked process constantly monitors the test logs and kills the test run if a UBSAN error is found. Change-Id: I783243458ac2765a97a1dd7dd40d458cc2e1d80b Reviewed-on: http://gerrit.cloudera.org:8080/11813 Reviewed-by: Jim Apple <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f79f2d4a Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f79f2d4a Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f79f2d4a Branch: refs/heads/branch-3.1.0 Commit: f79f2d4ad0b8ea2f209785a9b2bec61703102881 Parents: 87c2ad0 Author: Jim Apple <[email protected]> Authored: Sat Oct 27 21:47:54 2018 -0700 Committer: Zoltan Borok-Nagy <[email protected]> Committed: Tue Nov 13 12:52:36 2018 +0100 ---------------------------------------------------------------------- bin/jenkins/all-tests.sh | 34 ++++++++++++++++++++++++++++++++++ bin/jenkins/finalize.sh | 9 +++++++++ 2 files changed, 43 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/f79f2d4a/bin/jenkins/all-tests.sh ---------------------------------------------------------------------- diff --git a/bin/jenkins/all-tests.sh b/bin/jenkins/all-tests.sh index d29f383..b531804 100644 --- a/bin/jenkins/all-tests.sh +++ b/bin/jenkins/all-tests.sh @@ -26,6 +26,40 @@ cd "${IMPALA_HOME}" export IMPALA_MAVEN_OPTIONS="-U" +# When UBSAN_FAIL is "death", the logs are monitored for UBSAN errors. Any errors will +# then cause this script to exit. +# +# When UBSAN_FAIL is "error", monitoring is delayed until tests have finished running. +# +# Any other value ignores UBSAN errors. +: ${UBSAN_FAIL:=error} +export UBSAN_FAIL + +if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \ + && [ "${UBSAN_FAIL}" = "death" ] +then + export PID_TO_KILL="$(echo $$)" + mkdir -p "${IMPALA_HOME}/logs" + + function killer { + while ! grep -rI ": runtime error: " "${IMPALA_HOME}/logs" + do + sleep 1 + if ! test -e "/proc/$PID_TO_KILL" + then + return + fi + done + >&2 echo "Killing process $PID_TO_KILL because it invoked undefined behavior" + kill -9 $PID_TO_KILL + } + + killer & + export KILLER_PID="$(echo $!)" + disown + trap "kill -i $KILLER_PID" EXIT +fi + source bin/bootstrap_development.sh RET_CODE=0 http://git-wip-us.apache.org/repos/asf/impala/blob/f79f2d4a/bin/jenkins/finalize.sh ---------------------------------------------------------------------- diff --git a/bin/jenkins/finalize.sh b/bin/jenkins/finalize.sh index aaebb83..b7adfb4 100755 --- a/bin/jenkins/finalize.sh +++ b/bin/jenkins/finalize.sh @@ -21,6 +21,15 @@ set -euo pipefail trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \ + && [ "${UBSAN_FAIL}" = "error" ] \ + && { grep -rI ": runtime error: " "${IMPALA_HOME}/logs" 2>&1 | sort | uniq \ + | tee logs/ubsan.txt ; } +then + "${IMPALA_HOME}"/bin/generate_junitxml.py --step UBSAN \ + --stderr "${IMPALA_HOME}"/logs/ubsan.txt --error "Undefined C++ behavior" +fi + rm -rf "${IMPALA_HOME}"/logs_system mkdir -p "${IMPALA_HOME}"/logs_system dmesg > "${IMPALA_HOME}"/logs_system/dmesg
