Add a build flag for the undefined behavior sanitizer, aka "ubsan".
Ubsan checks for undefined behavior according to the C++ standard. Some of this behavior has been known to be exploited by optimizing compilers to produce bizarre results, like taking both branches of a conditional. This patch only adds build options; fixing the errors ubsan finds, as well as adding any tests that a build is free from ubsan errors, are not covered in this patch. Change-Id: I03044c657ac171daa0648f833bbbeed7bdde49cb Reviewed-on: http://gerrit.cloudera.org:8080/6186 Reviewed-by: Dan Hecht <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/93eb8ccb Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/93eb8ccb Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/93eb8ccb Branch: refs/heads/master Commit: 93eb8ccbedda060af01dccbb452c28d8f2072115 Parents: 1417d76 Author: Jim Apple <[email protected]> Authored: Tue Feb 28 12:01:13 2017 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Mar 1 03:09:17 2017 +0000 ---------------------------------------------------------------------- CMakeLists.txt | 3 ++- be/CMakeLists.txt | 21 +++++++++++++++++++-- bin/make_impala.sh | 3 ++- bin/run-backend-tests.sh | 3 +++ bin/start-impalad.sh | 2 ++ buildall.sh | 8 ++++++++ 6 files changed, 36 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 80256cc..207fac4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,7 +201,8 @@ find_package(LlvmBinaries REQUIRED) # Find LLVM libraries to link against. if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" - OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY") + OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY" + OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN") # Use the LLVM libaries with assertions for debug builds. set(LLVM_ROOT ${LLVM_DEBUG_ROOT}) endif() http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/be/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 3697ceb..1e094a0 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -87,6 +87,19 @@ SET(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG -gdwarf-2") SET(CXX_FLAGS_ADDRESS_SANITIZER "${CXX_CLANG_FLAGS} -O1 -g -fsanitize=address -fno-omit-frame-pointer -DADDRESS_SANITIZER") +# Set the flags to the undefined behavior sanitizer, also known as "ubsan" +# Turn on sanitizer and debug symbols to get stack traces: +SET(CXX_FLAGS_UBSAN "${CXX_CLANG_FLAGS} -ggdb3 -fno-omit-frame-pointer -fsanitize=undefined") +# Add flags to enable symbol resolution in the stack traces: +SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -rtlib=compiler-rt -lgcc_s") +# Ignore a number of noisy errors with too many false positives: +SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -fno-sanitize=alignment,function,vptr,float-divide-by-zero,float-cast-overflow") +# Don't enforce wrapped signed integer arithmetic so that the sanitizer actually sees +# undefined wrapping: +SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -fno-wrapv") +# To ease debugging, turn off all optimizations: +SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -O0") + SET(CXX_FLAGS_TIDY "${CXX_CLANG_FLAGS}") # Catching unused variables requires an optimization level greater than 0 SET(CXX_FLAGS_TIDY "${CXX_FLAGS_TIDY} -O1") @@ -111,6 +124,8 @@ elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER") SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_ADDRESS_SANITIZER}") elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TIDY") SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_TIDY}") +elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN") + SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_UBSAN}") else() message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") endif() @@ -133,7 +148,8 @@ if (CCACHE AND NOT DEFINED ENV{DISABLE_CCACHE}) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) if ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" - OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY") + OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY" + OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN") # Need to set CCACHE_CPP so that ccache calls clang with the original source file for # both preprocessing and compilation. Otherwise, ccache will use clang to preprocess # the file and then call clang with the preprocessed output if not cached. However, @@ -247,7 +263,8 @@ add_definitions(-fPIC) # set compile output directory if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR - "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER") + "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" OR + "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN") set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/debug/") else() set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/release/") http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/make_impala.sh ---------------------------------------------------------------------- diff --git a/bin/make_impala.sh b/bin/make_impala.sh index 70d088f..8d164c2 100755 --- a/bin/make_impala.sh +++ b/bin/make_impala.sh @@ -148,7 +148,8 @@ then if [[ ! -z $IMPALA_TOOLCHAIN ]]; then if [[ ("$TARGET_BUILD_TYPE" == "ADDRESS_SANITIZER") \ - || ("$TARGET_BUILD_TYPE" == "TIDY") ]] + || ("$TARGET_BUILD_TYPE" == "TIDY") \ + || ("$TARGET_BUILD_TYPE" == "UBSAN") ]] then CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/clang_toolchain.cmake) else http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/run-backend-tests.sh ---------------------------------------------------------------------- diff --git a/bin/run-backend-tests.sh b/bin/run-backend-tests.sh index 758c1bc..f426542 100755 --- a/bin/run-backend-tests.sh +++ b/bin/run-backend-tests.sh @@ -34,6 +34,9 @@ fi cd ${IMPALA_BE_DIR} . ${IMPALA_HOME}/bin/set-classpath.sh +cd .. export CTEST_OUTPUT_ON_FAILURE=1 +export UBSAN_OPTIONS="print_stacktrace=1" +export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}" make test ARGS="${BE_TEST_ARGS}" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/start-impalad.sh ---------------------------------------------------------------------- diff --git a/bin/start-impalad.sh b/bin/start-impalad.sh index e7f2442..a731804 100755 --- a/bin/start-impalad.sh +++ b/bin/start-impalad.sh @@ -103,4 +103,6 @@ if ${CLUSTER_DIR}/admin is_kerberized; then fi . ${IMPALA_HOME}/bin/set-classpath.sh +export UBSAN_OPTIONS='print_stacktrace=1' +export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}" exec ${TOOL_PREFIX} ${IMPALA_CMD} ${IMPALAD_ARGS} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/buildall.sh ---------------------------------------------------------------------- diff --git a/buildall.sh b/buildall.sh index 9c4385c..bcc6ae3 100755 --- a/buildall.sh +++ b/buildall.sh @@ -59,6 +59,7 @@ CODE_COVERAGE=0 BUILD_ASAN=0 BUILD_FE_ONLY=0 BUILD_TIDY=0 +BUILD_UBSAN=0 # Export MAKE_CMD so it is visible in scripts that invoke make, e.g. copy-udfs-udas.sh export MAKE_CMD=make LZO_CMAKE_ARGS= @@ -114,6 +115,9 @@ do -tidy) BUILD_TIDY=1 ;; + -ubsan) + BUILD_UBSAN=1 + ;; -testpairwise) EXPLORATION_STRATEGY=pairwise ;; @@ -182,6 +186,7 @@ do echo "[-codecoverage] : Build with code coverage [Default: False]" echo "[-asan] : Address sanitizer build [Default: False]" echo "[-tidy] : clang-tidy build [Default: False]" + echo "[-ubsan] : Undefined behavior build [Default: False]" echo "[-skiptests] : Skips execution of all tests" echo "[-notests] : Skips building and execution of all tests" echo "[-start_minicluster] : Start test cluster including Impala and all"\ @@ -260,6 +265,9 @@ fi if [[ ${BUILD_TIDY} -eq 1 ]]; then CMAKE_BUILD_TYPE=TIDY fi +if [[ ${BUILD_UBSAN} -eq 1 ]]; then + CMAKE_BUILD_TYPE=UBSAN +fi MAKE_IMPALA_ARGS+=" -build_type=${CMAKE_BUILD_TYPE}"
