IMPALA-3223: Removal of non-toolchain builds. This change removes the option to build without specifying the environment variable $IMPALA_TOOLCHAIN. By default, if it's not set, sourcing impala-config.sh will set it to $IMPALA_HOME/toolchain. A user can override it by setting $IMPALA_TOOLCHAIN to his/her own toolchain directory. The user can also set $SKIP_TOOLCHAIN_BOOTSTRAP to true to avoid running the toolchain bootstrapping script (e.g. a particular component in toolchain is at a version not checked into S3).
$IMPALA_TOOLCHAIN holds some third party binaries which Impala relies on. They can be compiled from source in the native toolchain which is public. This commit also removes build_thirdparty.sh as it's no longer used. By default, Impala will be built with the compiler in $IMPALA_TOOLCHAIN but this option can be overridden by setting environment variable $USE_SYSTEM_GCC to 1. Change-Id: I42b60e99fb9caf1294be7ab242856ca3b9a5ab73 Reviewed-on: http://gerrit.cloudera.org:8080/3259 Reviewed-by: Michael Ho <[email protected]> Tested-by: Michael Ho <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/86ff18ee Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/86ff18ee Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/86ff18ee Branch: refs/heads/master Commit: 86ff18eee9cf3babe0d596050088bf75b5287467 Parents: 2fb14d1 Author: Michael Ho <[email protected]> Authored: Fri May 27 19:19:36 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Tue Jun 7 17:29:59 2016 -0700 ---------------------------------------------------------------------- CMakeLists.txt | 148 +++++++++----------- be/CMakeLists.txt | 38 ++--- bin/build_thirdparty.sh | 241 -------------------------------- bin/impala-config.sh | 109 ++++++--------- buildall.sh | 14 +- cmake_modules/FindPICLibs.cmake | 19 --- cmake_modules/asan_toolchain.cmake | 2 - cmake_modules/toolchain.cmake | 2 - 8 files changed, 116 insertions(+), 457 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index dc0ec57..0c09ccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,28 +14,19 @@ cmake_minimum_required(VERSION 2.6) -if (NOT $ENV{DISABLE_IMPALA_TOOLCHAIN}) - # Enable toolchain variable if the environment is setup - set(IMPALA_TOOLCHAIN ON) - message(STATUS "Toolchain build.") - if ("$ENV{SKIP_TOOLCHAIN_BOOTSTRAP}" STREQUAL "true") - message(STATUS "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap.") +if ("$ENV{SKIP_TOOLCHAIN_BOOTSTRAP}" STREQUAL "true") + message(STATUS "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap.") +else() + # Download any missing toolchain dependencies. If this fails, fail the build. + set(BOOTSTRAP_CMD "$ENV{IMPALA_HOME}/bin/bootstrap_toolchain.py") + # Download and unpack the dependencies + message(STATUS "Downloading and extracting dependencies.") + execute_process(COMMAND ${BOOTSTRAP_CMD} RESULT_VARIABLE BOOTSTRAP_RESULT) + if (${BOOTSTRAP_RESULT} EQUAL 0) + message(STATUS "Toolchain bootstrap complete.") else() - # Download any missing toolchain dependencies. If this fails, fail the build. - set(BOOTSTRAP_CMD "$ENV{IMPALA_HOME}/bin/bootstrap_toolchain.py") - # Download and unpack the dependencies - message(STATUS "Downloading and extracting dependencies.") - execute_process(COMMAND ${BOOTSTRAP_CMD} RESULT_VARIABLE BOOTSTRAP_RESULT) - if (${BOOTSTRAP_RESULT} EQUAL 0) - message(STATUS "Toolchain bootstrap complete.") - else() - message(FATAL_ERROR "Toolchain bootstrap failed.") - endif() + message(FATAL_ERROR "Toolchain bootstrap failed.") endif() -else() - set(IMPALA_TOOLCHAIN OFF) - message(STATUS "Impala toolchain was explicitly disabled using DISABLE_IMPALA_TOOLCHAIN.") - message(STATUS "Assuming system search path for dependencies.") endif() # Explicitly define project() to allow modifying the compiler before the project is @@ -84,63 +75,51 @@ function(set_dep_root NAME) set(${NAME}_ROOT $ENV{IMPALA_TOOLCHAIN}/${NAME_LOWER}-$ENV{${VAL_NAME}} PARENT_SCOPE) endfunction() -set(Boost_USE_MULTITHREADED ON) - -if (IMPALA_TOOLCHAIN) - message(STATUS "Found Impala Toolchain, enabling toolchain boost") - set(ENV{BOOST_ROOT} "$ENV{IMPALA_TOOLCHAIN}/boost-$ENV{IMPALA_BOOST_VERSION}") - set(Boost_USE_STATIC_LIBS NOT ${BUILD_SHARED_LIBS}) - set(Boost_USE_STATIC_RUNTIME ON) - - # Newer versions of boost (including the version in toolchain) don't build separate - # multithreaded versions (they always are). Make sure to pick those up. - set(Boost_USE_MULTITHREADED OFF) - - # Define root path for all dependencies, this is in the form of - # set_dep_root(PACKAGE) -> - # PACKAGE_ROOT set to $ENV{IMPALA_TOOLCHAIN}/PACKAGE-$ENV{IMPALA_PACKAGE_VERSION} - set_dep_root(GCC) - set_dep_root(AVRO) - set_dep_root(BREAKPAD) - set_dep_root(BZIP2) - set_dep_root(GFLAGS) - set_dep_root(GLOG) - set_dep_root(GPERFTOOLS) - set_dep_root(GTEST) - set_dep_root(LLVM) - set(LLVM_DEBUG_ROOT $ENV{IMPALA_TOOLCHAIN}/llvm-$ENV{IMPALA_LLVM_DEBUG_VERSION}) - set_dep_root(LZ4) - set_dep_root(OPENLDAP) - set_dep_root(RE2) - set_dep_root(RAPIDJSON) - set_dep_root(SNAPPY) - set_dep_root(THRIFT) - set_dep_root(ZLIB) - if (APPLE) - set_dep_root(OPENSSL) - endif() -endif () +# Define root path for all dependencies, this is in the form of +# set_dep_root(PACKAGE) -> +# PACKAGE_ROOT set to $ENV{IMPALA_TOOLCHAIN}/PACKAGE-$ENV{IMPALA_PACKAGE_VERSION} +set_dep_root(AVRO) +set_dep_root(BOOST) +set_dep_root(BREAKPAD) +set_dep_root(BZIP2) +set_dep_root(GCC) +set_dep_root(GFLAGS) +set_dep_root(GLOG) +set_dep_root(GPERFTOOLS) +set_dep_root(GTEST) +set_dep_root(LLVM) +set(LLVM_DEBUG_ROOT $ENV{IMPALA_TOOLCHAIN}/llvm-$ENV{IMPALA_LLVM_DEBUG_VERSION}) +set_dep_root(LZ4) +set_dep_root(OPENLDAP) +set_dep_root(RE2) +set_dep_root(RAPIDJSON) +set_dep_root(SNAPPY) +set_dep_root(THRIFT) +set_dep_root(ZLIB) +if (APPLE) + set_dep_root(OPENSSL) +endif() + +set(Boost_USE_STATIC_LIBS NOT ${BUILD_SHARED_LIBS}) +set(Boost_USE_STATIC_RUNTIME ON) + +# Newer versions of boost (including the version in toolchain) don't build separate +# multithreaded versions (they always are). Make sure to pick those up. +# TODO: understand the consequence of leaving this ON (the default value). +set(Boost_USE_MULTITHREADED OFF) -# find libraries that will be statically linked if they were compiled with -fPIC -find_package(PICLibs REQUIRED) - -# find boost headers and libs -IF (DEFINED ENV{BOOST_ROOT}) - # The casing and underscoring expected for these properties varies between - # versions of CMake. Multiple inconsistent versions may be present here - # intentionally to provide what a wide range of versions expects. - set(Boost_NO_SYSTEM_PATHS true) - set(BOOST_ROOT $ENV{BOOST_ROOT}) - set(BOOST_LIBRARYDIR $ENV{BOOST_ROOT}/lib) - set(BOOST_INCLUDEDIR $ENV{BOOST_ROOT}/include) - set(Boost_INCLUDE_DIR ${BOOST_INCLUDEDIR}) -ENDIF (DEFINED ENV{BOOST_ROOT}) +# The casing and underscoring expected for these properties varies between +# versions of CMake. Multiple inconsistent versions may be present here +# intentionally to provide what a wide range of versions expects. +set(Boost_NO_SYSTEM_PATHS true) +set(BOOST_LIBRARYDIR ${BOOST_ROOT}/lib) +set(BOOST_INCLUDEDIR ${BOOST_ROOT}/include) +set(Boost_INCLUDE_DIR ${BOOST_INCLUDEDIR}) if (CMAKE_DEBUG) set(Boost_DEBUG TRUE) endif() - find_package(Boost $ENV{IMPALA_BOOST_VERSION} REQUIRED COMPONENTS thread regex system filesystem date_time) include_directories(${Boost_INCLUDE_DIRS}) @@ -160,20 +139,17 @@ find_package(Zlib REQUIRED) include_directories(${ZLIB_INCLUDE_DIR}) if(NOT BUILD_SHARED_LIBS) - # It might be the case that bzip2 and zlib are not available as static libs in the non - # toolchain case - # TODO remove this when moving to toolchain only + # Panic if we cannot find the static libraries as they are supposed to be + # in the toolchain. if (NOT BZIP2_STATIC_FOUND) - set(LIBBZ2 ${BZIP2_LIBRARIES}) - else() - set(LIBBZ2 BZIP2_STATIC) + message(FATAL_ERROR "bzip2 static library libbz2.a not found in $ENV{IMPALA_TOOLCHAIN}") endif() - if (NOT ZLIB_STATIC_FOUND) - set(LIBZ ${ZLIB_LIBRARIES}) - else() - set(LIBZ ZLIB_STATIC) + message(FATAL_ERROR "zlib static library libz.a not found in $ENV{IMPALA_TOOLCHAIN}") endif() + + set(LIBBZ2 BZIP2_STATIC) + set(LIBZ ZLIB_STATIC) else() set(LIBBZ2 ${BZIP2_LIBRARIES}) set(LIBZ ${ZLIB_LIBRARIES}) @@ -226,8 +202,8 @@ set(LLVM_BINARIES_ROOT ${LLVM_ROOT}) find_package(LlvmBinaries REQUIRED) # Find LLVM libraries to link against. -if (IMPALA_TOOLCHAIN AND ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" - OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")) +if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" + OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER") # Use the LLVM libaries with assertions for debug builds. set(LLVM_ROOT ${LLVM_DEBUG_ROOT}) endif() @@ -253,6 +229,12 @@ set(LDAP_LIBRARY ldapstatic) set(LBER_LIBRARY lberstatic) message(STATUS "LDAP: ${LDAP_INCLUDE_DIR}") +# The environment variable $THRIFT_HOME is set in impala-config.sh +# Make sure it's consistent with $THRIFT_ROOT. +if (NOT ($ENV{THRIFT_HOME} STREQUAL ${THRIFT_ROOT})) + message(FATAL_ERROR "THRIFT_ROOT (${THRIFT_ROOT}) differs from environment " + "variable THRIFT_HOME ($ENV{THRIFT_HOME}).") +endif() # find thrift headers and libs find_package(Thrift REQUIRED) include_directories(${THRIFT_INCLUDE_DIR}) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/be/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index ed4a8c4..5d3e246 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -43,9 +43,7 @@ SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG") IF($ENV{USE_GOLD_LINKER} STREQUAL "true") SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wl,-fuse-ld=gold") - if (IMPALA_TOOLCHAIN) - SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -B $ENV{IMPALA_TOOLCHAIN}/binutils-$ENV{IMPALA_BINUTILS_VERSION}/bin/") - endif() + SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -B $ENV{IMPALA_TOOLCHAIN}/binutils-$ENV{IMPALA_BINUTILS_VERSION}/bin/") ENDIF() # On Apple we build with clang and need libstdc++ instead of libc++ @@ -78,11 +76,9 @@ SET(CXX_GCC_FLAGS "-g -Wno-unused-local-typedefs") # # Debug information is stored as dwarf2 to be as compatible as possible SET(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -ggdb -gdwarf-2") -if (IMPALA_TOOLCHAIN) - # -Werror: compile warnings should be errors when using the toolchain compiler. - # Only enable for debug builds because this is what we test in pre-commit tests. - SET(CXX_FLAGS_DEBUG "${CXX_FLAGS_DEBUG} -Werror") -endif() +# -Werror: compile warnings should be errors when using the toolchain compiler. +# Only enable for debug builds because this is what we test in pre-commit tests. +SET(CXX_FLAGS_DEBUG "${CXX_FLAGS_DEBUG} -Werror") SET(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG -gdwarf-2") SET(CXX_FLAGS_ADDRESS_SANITIZER "${CXX_CLANG_FLAGS} -O1 -g -fsanitize=address -fno-omit-frame-pointer -DADDRESS_SANITIZER") @@ -93,9 +89,6 @@ if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") SET(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE}) elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER") - if (NOT IMPALA_TOOLCHAIN) - SET(CMAKE_CXX_COMPILER ${LLVM_CLANG_EXECUTABLE}) - endif() SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_ADDRESS_SANITIZER}") else() message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") @@ -145,10 +138,9 @@ set(CLANG_IR_CXX_FLAGS "-emit-llvm" "-c" "-std=c++14" "-DIR_COMPILE" "-DNDEBUG" "-DHAVE_INTTYPES_H" "-DHAVE_NETINET_IN_H" "-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG" "-DBOOST_NO_EXCEPTIONS" "-fcolor-diagnostics" "-Wno-deprecated" "-Wno-return-type-c-linkage" "-O1") -if (IMPALA_TOOLCHAIN) - # -Werror: compile warnings should be errors when using the toolchain compiler. - SET(CLANG_IR_CXX_FLAGS "${CLANG_IR_CXX_FLAGS}" "-Werror") -endif() +# -Werror: compile warnings should be errors when using the toolchain compiler. +set(CLANG_IR_CXX_FLAGS "${CLANG_IR_CXX_FLAGS}" "-Werror") + IF($ENV{ENABLE_IMPALA_IR_DEBUG_INFO} STREQUAL "true") # -g: emit debug symbols in IR. These increase IR size and memory overhead of LLVM, but # are useful for debugging codegened code and interpreting codegen disassembly @@ -160,7 +152,6 @@ endif() # -inline: inline with low threshold to get rid of trivial accessor functions. set(LLVM_OPT_IR_FLAGS "-inline" "-inlinehint-threshold=10" "-inline-threshold=10") - # setup doc generation with Doxygen find_package(Doxygen) if (DOXYGEN_FOUND) @@ -203,11 +194,9 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/generated-sources) set(CLANG_INCLUDE_FLAGS) -if (IMPALA_TOOLCHAIN) - # Ensure that clang uses the gcc toolchain headers. - set(CLANG_BASE_FLAGS --gcc-toolchain=${GCC_ROOT}) - set(CLANG_INCLUDE_FLAGS ${CLANG_BASE_FLAGS}) -endif() +# Ensure that clang uses the gcc toolchain headers. +set(CLANG_BASE_FLAGS --gcc-toolchain=${GCC_ROOT}) +set(CLANG_INCLUDE_FLAGS ${CLANG_BASE_FLAGS}) set(CLANG_INCLUDE_FLAGS ${CLANG_INCLUDE_FLAGS} @@ -355,13 +344,6 @@ endif() set(IMPALA_LINK_LIBS_DYNAMIC_TARGETS ${IMPALA_LINK_LIBS_NO_TCMALLOC}) list(REMOVE_ITEM IMPALA_LINK_LIBS_DYNAMIC_TARGETS HDFS_STATIC) set(IMPALA_LINK_LIBS_DYNAMIC_TARGETS ${IMPALA_LINK_LIBS_DYNAMIC_TARGETS} ${HDFS_LIBRARIES}) -# Replace static libz bzib with dynamic -if (NOT IMPALA_TOOLCHAIN AND NOT BUILD_SHARED_LIBS) - list(REMOVE_ITEM IMPALA_LINK_LIBS_DYNAMIC_TARGETS BZIP2_STATIC) - list(REMOVE_ITEM IMPALA_LINK_LIBS_DYNAMIC_TARGETS ZLIB_STATIC) - set(IMPALA_LINK_LIBS_DYNAMIC_TARGETS - ${IMPALA_LINK_LIBS_DYNAMIC_TARGETS} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES}) -endif() # Link libs for test executables. Although not all tests need all libs, # the build time for the tests is rather small and not worth the trouble. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/bin/build_thirdparty.sh ---------------------------------------------------------------------- diff --git a/bin/build_thirdparty.sh b/bin/build_thirdparty.sh deleted file mode 100755 index ce1f323..0000000 --- a/bin/build_thirdparty.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2012 Cloudera Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cleans and rebuilds thirdparty/. The Impala build environment must be set up -# by bin/impala-config.sh before running this script. - -# Exit on non-true return value -set -e -# Exit on reference to uninitialized variable -set -u - -# By default, git clean every library we build -CLEAN_ACTION=1 - -# By default, build every library -BUILD_ALL=1 - -# If BUILD_ALL -eq 1, these are ignored, otherwise only build those libraries with -# BUILD_<lib> -eq 1 -BUILD_AVRO=0 -BUILD_THRIFT=0 -BUILD_GLOG=0 -BUILD_GFLAGS=0 -BUILD_GTEST=0 -BUILD_RE2=0 -BUILD_SASL=0 -BUILD_LDAP=0 -BUILD_SNAPPY=0 -BUILD_PPROF=0 -BUILD_CDH4EXTRAS=0 -BUILD_LZ4=0 - -for ARG in $* -do - case "$ARG" in - -noclean) - CLEAN_ACTION=0 - ;; - -avro) - BUILD_ALL=0 - BUILD_AVRO=1 - ;; - -glog) - BUILD_ALL=0 - BUILD_GLOG=1 - ;; - -thrift) - BUILD_ALL=0 - BUILD_THRIFT=1 - ;; - -gflags) - BUILD_ALL=0 - BUILD_GFLAGS=1 - ;; - -gtest) - BUILD_ALL=0 - BUILD_GTEST=1 - ;; - -re2) - BUILD_ALL=0 - BUILD_RE2=1 - ;; - -sasl) - BUILD_ALL=0 - BUILD_SASL=1 - ;; - -ldap) - BUILD_ALL=0 - BUILD_LDAP=1 - ;; - -snappy) - BUILD_ALL=0 - BUILD_SNAPPY=1 - ;; - -lz4) - BUILD_ALL=0 - BUILD_LZ4=1 - ;; - -pprof) - BUILD_ALL=0 - BUILD_PPROF=1 - ;; - -*) - echo "Usage: build_thirdparty.sh [-noclean] \ -[-avro -glog -thrift -gflags -gtest -re2 -sasl -ldap -snappy -pprof]" - exit 1 - esac -done -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` -. "$bin"/impala-config.sh -USE_PIC_LIB_PATH=${PIC_LIB_PATH:-} - -function build_preamble() { - echo - echo - echo "********************************************************************************" - echo "Building $2 in $1 $([ $CLEAN_ACTION -eq 1 ] && echo '(clean)')" - echo "********************************************************************************" - cd $1 - if [ $CLEAN_ACTION -eq 1 ]; then - # remove everything that is not checked in - git clean -dfx - fi -} - -# Build Sasl -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_SASL -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/cyrus-sasl-${IMPALA_CYRUS_SASL_VERSION} Sasl - - # Need to specify which libdb to use on certain OSes - LIBDB_DIR="" - if [[ -e "/usr/lib64/libdb4" && -e "/usr/include/libdb4" ]]; then - LIBDB_DIR="--with-bdb-libdir=/usr/lib64/libdb4 --with-bdb-incdir=/usr/include/libdb4" - fi - # Disable everything except those protocols needed -- currently just Kerberos. - # Sasl does not have a --with-pic configuration. - CFLAGS="-fPIC -DPIC" CXXFLAGS="-fPIC -DPIC" ./configure \ - --disable-sql --disable-otp --disable-ldap --disable-digest --with-saslauthd=no \ - --prefix=$IMPALA_CYRUS_SASL_INSTALL_DIR --enable-static --enable-staticdlopen \ - $LIBDB_DIR - # the first time you do a make it fails, build again. - (make || make) - make install -fi - -set -e -# build thrift -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_THRIFT -eq 1 ]; then - cd ${THRIFT_SRC_DIR} - build_preamble ${THRIFT_SRC_DIR} "Thrift" - if [ -d "${PIC_LIB_PATH:-}" ]; then - PIC_LIB_OPTIONS="--with-zlib=${PIC_LIB_PATH} " - fi - JAVA_PREFIX=${THRIFT_HOME}/java PY_PREFIX=${THRIFT_HOME}/python \ - ./configure --with-pic --prefix=${THRIFT_HOME} \ - --with-php=no --with-java=no --with-perl=no --with-erlang=no \ - --with-ruby=no --with-haskell=no --with-erlang=no --with-d=no \ - --with-go=no --with-qt4=no --with-libevent=no ${PIC_LIB_OPTIONS:-} - make # Make with -j fails - make install - cd ${THRIFT_SRC_DIR}/contrib/fb303 - chmod 755 ./bootstrap.sh - ./bootstrap.sh - chmod 755 configure - CPPFLAGS="-I${THRIFT_HOME}/include" PY_PREFIX=${THRIFT_HOME}/python ./configure \ - --with-java=no --with-php=no --prefix=${THRIFT_HOME} --with-thriftpath=${THRIFT_HOME} - make - make install -fi - -# build gflags -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_GFLAGS -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/gflags-${IMPALA_GFLAGS_VERSION} GFlags - GFLAGS_INSTALL=`pwd`/third-party-install - ./configure --with-pic --prefix=${GFLAGS_INSTALL} - make -j${IMPALA_BUILD_THREADS:-4} install -fi - -# Build pprof -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_PPROF -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/gperftools-${IMPALA_GPERFTOOLS_VERSION} \ - GPerftools - # TODO: google perf tools indicates this might be necessary on 64 bit systems. - # we're not compiling the rest of our code to not omit frame pointers but it - # still seems to generate useful profiling data. - ./configure --enable-frame-pointers --with-pic - make -j${IMPALA_BUILD_THREADS:-4} -fi - -# Build glog -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_GLOG -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/glog-${IMPALA_GLOG_VERSION} GLog - ./configure --with-pic --with-gflags=${GFLAGS_INSTALL} - # SLES's gcc45-c++ is required for sse2 support (default is 4.3), but crashes - # when building logging_unittest-logging_unittest.o. Telling it to uses the - # stabs format for debugging symbols instead of dwarf exercises a different - # code path to work around this issue. - cat > Makefile.gcc45sles_workaround <<EOF -logging_unittest-logging_unittest.o : CXXFLAGS= -gstabs -O2 -EOF - cat Makefile >> Makefile.gcc45sles_workaround - mv Makefile.gcc45sles_workaround Makefile - make -j${IMPALA_BUILD_THREADS:-4} -fi - -# Build gtest -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_GTEST -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/gtest-${IMPALA_GTEST_VERSION} GTest - cmake . - make -j${IMPALA_BUILD_THREADS:-4} -fi - -# Build Snappy -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_SNAPPY -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/snappy-${IMPALA_SNAPPY_VERSION} Snappy - ./autogen.sh - ./configure --with-pic --prefix=$IMPALA_HOME/thirdparty/snappy-${IMPALA_SNAPPY_VERSION}/build - make install -fi - -# Build Lz4 -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_LZ4 -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/lz4 Lz4 - cmake . - make -fi - -# Build re2 -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_RE2 -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/re2 RE2 - make -j${IMPALA_BUILD_THREADS:-4} -fi - -# Build Ldap -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_LDAP -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/openldap-${IMPALA_OPENLDAP_VERSION} Openldap - ./configure --enable-slapd=no --prefix=`pwd`/impala_install --enable-static --with-pic - make -j${IMPALA_BUILD_THREADS:-4} - make -j${IMPALA_BUILD_THREADS:-4} depend - make install -fi - -# Build Avro -if [ $BUILD_ALL -eq 1 ] || [ $BUILD_AVRO -eq 1 ]; then - build_preamble $IMPALA_HOME/thirdparty/avro-c-${IMPALA_AVRO_VERSION} Avro - cmake . - make -j${IMPALA_BUILD_THREADS:-4} -fi http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 89b7400..d6a0833 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -21,7 +21,7 @@ # this script because scripts outside this repository may need to be updated and that # is not practical at this time. export JAVA_HOME="${JAVA_HOME:-/usr/java/default}" -if [ ! -d "$JAVA_HOME" ] ; then +if [ ! -d "$JAVA_HOME" ]; then echo "JAVA_HOME must be set to the location of your JDK!" return 1 fi @@ -39,8 +39,15 @@ if [ -z $IMPALA_HOME ]; then fi fi -: ${DISABLE_IMPALA_TOOLCHAIN=0} : ${IMPALA_TOOLCHAIN=$IMPALA_HOME/toolchain} +if [ -z $IMPALA_TOOLCHAIN ]; then + echo "IMPALA_TOOLCHAIN must be specified. Please set it to a valid directory or"\ + "leave it unset." + return 1 +fi + +# This flag is used in $IMPALA_HOME/cmake_modules/toolchain.cmake. +# If it's 0, Impala will be built with the compiler in the toolchain directory. : ${USE_SYSTEM_GCC=0} # Gold is available on newer systems and a full build with static linking is ~1.5 mins @@ -48,15 +55,14 @@ fi : ${USE_GOLD_LINKER=false} # Override the default compiler by setting a path to the new compiler. The default -# compiler depends on USE_SYSTEM_GCC, DISABLE_IMPALA_TOOLCHAIN, and IMPALA_GCC_VERSION. -# The intended use case is to set the compiler to distcc, in that case the user would -# also set IMPALA_BUILD_THREADS to increase parallelism. +# compiler depends on USE_SYSTEM_GCC and IMPALA_GCC_VERSION. The intended use case +# is to set the compiler to distcc, in that case the user would also set +# IMPALA_BUILD_THREADS to increase parallelism. : ${IMPALA_CXX_COMPILER=default} # If enabled, debug symbols are added to cross-compiled IR. : ${ENABLE_IMPALA_IR_DEBUG_INFO=false} -export DISABLE_IMPALA_TOOLCHAIN export IMPALA_TOOLCHAIN export USE_SYSTEM_GCC export USE_GOLD_LINKER @@ -95,9 +101,7 @@ export USE_KUDU_DEBUG_BUILD if [[ -z "${KUDU_IS_SUPPORTED-}" ]]; then KUDU_IS_SUPPORTED=true if [[ -z $KUDU_BUILD_DIR ]]; then - if [[ $DISABLE_IMPALA_TOOLCHAIN -eq 1 ]]; then - KUDU_IS_SUPPORTED=false - elif ! $IS_OSX; then + if ! $IS_OSX; then if ! which lsb_release &>/dev/null; then echo Unable to find the 'lsb_release' command. \ Please ensure it is available in your PATH. 1>&2 @@ -226,47 +230,35 @@ export NUM_CONCURRENT_TESTS=${NUM_CONCURRENT_TESTS-${CORES}} # Versions of toolchain dependencies (or if toolchain is not used of dependencies in # thirdparty) -export IMPALA_AVRO_VERSION=1.7.4 +export IMPALA_AVRO_VERSION=1.7.4-p4 export IMPALA_BINUTILS_VERSION=2.26 export IMPALA_BOOST_VERSION=1.57.0 export IMPALA_BREAKPAD_VERSION=20150612-p1 -export IMPALA_BZIP2_VERSION=1.0.6 +export IMPALA_BZIP2_VERSION=1.0.6-p1 export IMPALA_CYRUS_SASL_VERSION=2.1.23 export IMPALA_GCC_VERSION=4.9.2 export IMPALA_GFLAGS_VERSION=2.0 -export IMPALA_GLOG_VERSION=0.3.2 +export IMPALA_GLOG_VERSION=0.3.2-p1 export IMPALA_GPERFTOOLS_VERSION=2.5 export IMPALA_GTEST_VERSION=1.6.0 export IMPALA_KUDU_VERSION=0.8.0-RC1 -export IMPALA_LLVM_VERSION=3.8.0 -export IMPALA_LLVM_DEBUG_VERSION=3.8.0 -export IMPALA_LLVM_ASAN_VERSION=3.8.0 +export IMPALA_LLVM_VERSION=3.8.0-p1 +export IMPALA_LLVM_ASAN_VERSION=3.8.0-p1 +# Debug builds should use the release+asserts build to get additional coverage. +# Don't use the LLVM debug build because the binaries are too large to distribute. +export IMPALA_LLVM_DEBUG_VERSION=3.8.0-asserts-p1 export IMPALA_LZ4_VERSION=svn export IMPALA_OPENLDAP_VERSION=2.4.25 export IMPALA_OPENSSL_VERSION=0.9.8zf export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=9.0-801 export IMPALA_RAPIDJSON_VERSION=0.11 -export IMPALA_RE2_VERSION=20130115 +export IMPALA_RE2_VERSION=20130115-p1 export IMPALA_SNAPPY_VERSION=1.1.3 export IMPALA_SQUEASEL_VERSION=3.3 -export IMPALA_THRIFT_VERSION=0.9.0 +export IMPALA_THRIFT_VERSION=0.9.0-p8 export IMPALA_THRIFT_JAVA_VERSION=0.9.0 export IMPALA_ZLIB_VERSION=1.2.8 -# Some of the variables need to be overwritten to explicitly mark the patch level -if [[ -n "$IMPALA_TOOLCHAIN" ]]; then - IMPALA_AVRO_VERSION+=-p4 - IMPALA_BZIP2_VERSION+=-p1 - IMPALA_GLOG_VERSION+=-p1 - IMPALA_THRIFT_VERSION+=-p8 - IMPALA_RE2_VERSION+=-p1 - IMPALA_LLVM_VERSION+=-p1 - IMPALA_LLVM_ASAN_VERSION+=-p1 - # Debug builds should use the release+asserts build to get additional coverage. - # Don't use the LLVM debug build because the binaries are too large to distribute. - IMPALA_LLVM_DEBUG_VERSION+=-asserts-p1 -fi - export KUDU_MASTER=${KUDU_MASTER:-"127.0.0.1"} export KUDU_MASTER_PORT=${KUDU_MASTER_PORT:-"7051"} # TODO: Figure out a way to use a snapshot version without causing a lot of breakage due @@ -282,20 +274,6 @@ if [[ $OSTYPE == "darwin"* ]]; then IMPALA_THRIFT_JAVA_VERSION=0.9.2 fi - -if [[ ! -z "${IMPALA_CYRUS_SASL_INSTALL_DIR:-}" ]] -then - export IMPALA_CYRUS_SASL_INSTALL_DIR # Ensure it's exported -elif [[ "${IMPALA_HOME}" =~ "~" ]] -then - # Sasl has problems with 'make install' if the path contains a ~, e.g. - # /some~directory/impala. In our packaging jobs, the path contains ~ so we'll - # just install somewhere else as a workaround. - export IMPALA_CYRUS_SASL_INSTALL_DIR=/tmp/impala-build/cyrus-sasl-${IMPALA_CYRUS_SASL_VERSION}/build -else - export IMPALA_CYRUS_SASL_INSTALL_DIR=${IMPALA_HOME}/thirdparty/cyrus-sasl-${IMPALA_CYRUS_SASL_VERSION}/build -fi - export IMPALA_HADOOP_VERSION=2.6.0-cdh5.8.0-SNAPSHOT export IMPALA_HBASE_VERSION=1.2.0-cdh5.8.0-SNAPSHOT export IMPALA_HIVE_VERSION=1.1.0-cdh5.8.0-SNAPSHOT @@ -313,8 +291,11 @@ export IMPALA_AUX_DATASET_DIR=$IMPALA_AUX_TEST_HOME/testdata/datasets export IMPALA_COMMON_DIR=$IMPALA_HOME/common export PATH=$IMPALA_HOME/bin:$PATH +# The directory in which all the thirdparty CDH components live. +CDH_COMPONENTS_HOME=$IMPALA_HOME/thirdparty + # Hadoop dependencies are snapshots in the Impala tree -export HADOOP_HOME=$IMPALA_HOME/thirdparty/hadoop-${IMPALA_HADOOP_VERSION}/ +export HADOOP_HOME=$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/ export HADOOP_CONF_DIR=$IMPALA_FE_DIR/src/test/resources : ${HADOOP_CLASSPATH=} @@ -326,12 +307,12 @@ HADOOP_CLASSPATH+=":$LZO_JAR_PATH" export MINI_DFS_BASE_DATA_DIR=$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data export PATH=$HADOOP_HOME/bin:$PATH -export LLAMA_HOME=$IMPALA_HOME/thirdparty/llama-${IMPALA_LLAMA_VERSION}/ -export MINIKDC_HOME=$IMPALA_HOME/thirdparty/llama-minikdc-${IMPALA_MINIKDC_VERSION} -export SENTRY_HOME=$IMPALA_HOME/thirdparty/sentry-${IMPALA_SENTRY_VERSION} +export LLAMA_HOME=$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/ +export MINIKDC_HOME=$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_MINIKDC_VERSION} +export SENTRY_HOME=$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION} export SENTRY_CONF_DIR=$IMPALA_HOME/fe/src/test/resources -export HIVE_HOME=$IMPALA_HOME/thirdparty/hive-${IMPALA_HIVE_VERSION}/ +export HIVE_HOME=$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/ export PATH=$HIVE_HOME/bin:$PATH export HIVE_CONF_DIR=$IMPALA_FE_DIR/src/test/resources @@ -346,7 +327,7 @@ export AUX_CLASSPATH="${LZO_JAR_PATH}" ### Tell hive not to use jline export HADOOP_USER_CLASSPATH_FIRST=true -export HBASE_HOME=$IMPALA_HOME/thirdparty/hbase-${IMPALA_HBASE_VERSION}/ +export HBASE_HOME=$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/ export PATH=$HBASE_HOME/bin:$PATH # Add the jars so hive can create hbase tables. @@ -358,18 +339,11 @@ export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA export HBASE_CONF_DIR=$HIVE_CONF_DIR -# Optionally set the Thrift home to the toolchain -if [[ -z $IMPALA_TOOLCHAIN ]]; then - THRIFT_SRC_DIR=${IMPALA_HOME}/thirdparty/thrift-${IMPALA_THRIFT_VERSION} - export THRIFT_HOME=${THRIFT_SRC_DIR}/build -else - export THRIFT_HOME=${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION} -fi +# Set $THRIFT_HOME to the Thrift directory in toolchain. +export THRIFT_HOME=${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION} # ASAN needs a matching version of llvm-symbolizer to symbolize stack traces. -if [[ -n $IMPALA_TOOLCHAIN ]]; then - export ASAN_SYMBOLIZER_PATH=${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer -fi +export ASAN_SYMBOLIZER_PATH=${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer export CLUSTER_DIR=${IMPALA_HOME}/testdata/cluster @@ -399,11 +373,7 @@ export LIBHDFS_OPTS="${LIBHDFS_OPTS}:${IMPALA_HOME}/be/build/debug/service" export ARTISTIC_STYLE_OPTIONS=$IMPALA_BE_DIR/.astylerc -if [[ -z "${IMPALA_TOOLCHAIN}" ]]; then - IMPALA_SNAPPY_PATH=${IMPALA_HOME}/thirdparty/snappy-${IMPALA_SNAPPY_VERSION}/build/lib -else - IMPALA_SNAPPY_PATH=${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib -fi +export IMPALA_SNAPPY_PATH=${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib export JAVA_LIBRARY_PATH=${IMPALA_SNAPPY_PATH} @@ -418,10 +388,11 @@ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JVM}`:`dirname ${LIB_HDFS}`" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_HOME}/be/build/debug/service" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_SNAPPY_PATH}" -LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$IMPALA_LZO/build" +LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_LZO}/build" -if [[ -n "$IMPALA_TOOLCHAIN" ]]; then - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64" +if [ $USE_SYSTEM_GCC -eq 0 ]; then + IMPALA_TOOLCHAIN_GCC_LIB=${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64 + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN_GCC_LIB}" fi export LD_LIBRARY_PATH http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/buildall.sh ---------------------------------------------------------------------- diff --git a/buildall.sh b/buildall.sh index 45df6cc..2c49b6f 100755 --- a/buildall.sh +++ b/buildall.sh @@ -244,16 +244,6 @@ if [ $CLEAN_ACTION -eq 1 ]; then $IMPALA_HOME/bin/clean.sh fi -# Only build thirdparty if no toolchain is set -if [[ -z $IMPALA_TOOLCHAIN ]]; then - # Sanity check that thirdparty is built. - if [ ! -e $IMPALA_HOME/thirdparty/gflags-${IMPALA_GFLAGS_VERSION}/libgflags.la ] - then - echo "Couldn't find thirdparty build files. Building thirdparty." - $IMPALA_HOME/bin/build_thirdparty.sh $([ ${CLEAN_ACTION} -eq 0 ] && echo '-noclean') - fi -fi - MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_type=${CMAKE_BUILD_TYPE}" if [ $BUILD_FE_ONLY -eq 1 ]; then @@ -308,9 +298,7 @@ $IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS} if [ -e $IMPALA_LZO ] then pushd $IMPALA_LZO - if [[ ! -z $IMPALA_TOOLCHAIN ]]; then - LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake" - fi + LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake" rm -f CMakeCache.txt cmake ${LZO_CMAKE_ARGS} ${MAKE_CMD} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/cmake_modules/FindPICLibs.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/FindPICLibs.cmake b/cmake_modules/FindPICLibs.cmake deleted file mode 100644 index edbf4fd..0000000 --- a/cmake_modules/FindPICLibs.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# In order to statically link in the Boost, bz2, event and z libraries, they -# needs to be recompiled with -fPIC. Set $PIC_LIB_PATH to the location of -# these libraries in the environment, or dynamic linking will be used instead. - -# If the toolchain is present, we have all the libraries compiled with -fpic, so we don't -# need this check - -if (NOT IMPALA_TOOLCHAIN) - IF (DEFINED ENV{PIC_LIB_PATH}) - set(CMAKE_SKIP_RPATH TRUE) - set(Boost_USE_STATIC_LIBS ON) - set(Boost_USE_STATIC_RUNTIME ON) - set(LIBBZ2 $ENV{PIC_LIB_PATH}/lib/libbz2.a) - set(LIBZ $ENV{PIC_LIB_PATH}/lib/libz.a) - ELSE (DEFINED ENV{PIC_LIB_PATH}) - set(Boost_USE_STATIC_LIBS OFF) - set(Boost_USE_STATIC_RUNTIME OFF) - ENDIF (DEFINED ENV{PIC_LIB_PATH}) -endif() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/cmake_modules/asan_toolchain.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/asan_toolchain.cmake b/cmake_modules/asan_toolchain.cmake index 05997bf..de6a338 100644 --- a/cmake_modules/asan_toolchain.cmake +++ b/cmake_modules/asan_toolchain.cmake @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(IMPALA_TOOLCHAIN ON) - # Set the root directory for the toolchain set(TOOLCHAIN_ROOT $ENV{IMPALA_TOOLCHAIN}) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/cmake_modules/toolchain.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/toolchain.cmake b/cmake_modules/toolchain.cmake index 86ebf9d..7cb5119 100644 --- a/cmake_modules/toolchain.cmake +++ b/cmake_modules/toolchain.cmake @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(IMPALA_TOOLCHAIN ON) - # Set the root directory for the toolchain set(TOOLCHAIN_ROOT $ENV{IMPALA_TOOLCHAIN})
