IMPALA-4653: fix sticky config variable problem Previously we could get a developer's shell into a bad state where a value of a config variable from a previous impala-config.sh version would override the value from the new impala-config.sh version.
This change adds a new mechanism to override settings locally by adding settings to impala-config-local.sh. This alternative approach is more robust, because the config variables will be reset to the intended values when impala-config.sh is re-sourced. impala-config-branch.sh can also be used to override settings in a version-controlled way, e.g. to support having different settings for different branches. I did not convert all variables to use this approach, since many people and Jenkins jobs depend on setting these variables from the environment. The remaining "sticky" variables are ones where default values should not change frequently, e.g. source directory locations and build settings. Change-Id: I930e2ca825142428d17a6981c77534ab0c8e3489 Reviewed-on: http://gerrit.cloudera.org:8080/5545 Reviewed-by: Matthew Jacobs <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/6b90aa3a Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/6b90aa3a Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/6b90aa3a Branch: refs/heads/master Commit: 6b90aa3a115f8274a5380945bc4e2b940ef4f711 Parents: 25ebf58 Author: Tim Armstrong <[email protected]> Authored: Fri Dec 16 11:10:09 2016 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Jan 5 01:43:36 2017 +0000 ---------------------------------------------------------------------- .gitignore | 1 + bin/impala-config-branch.sh | 24 +++++ bin/impala-config.sh | 210 +++++++++++++++++++++------------------ 3 files changed, 138 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6b90aa3a/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 9d5d981..ebd6bbe 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ org.eclipse.jdt.ui.prefs *benchmark_results.csv* load-*-generated.sql bin/version.info +bin/impala-config-local.sh # distcc options .impala_compiler_opts http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6b90aa3a/bin/impala-config-branch.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config-branch.sh b/bin/impala-config-branch.sh new file mode 100644 index 0000000..adad8cf --- /dev/null +++ b/bin/impala-config-branch.sh @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Variables in the file override the default values from impala-config.sh. +# Config changes for release or features branches should go here so they +# can be version controlled but not conflict with changes on the master +# branch. +# +# E.g. to override IMPALA_HADOOP_VERSION, you could uncomment this line: +# IMPALA_HADOOP_VERSION=3.0.0 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6b90aa3a/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index c9b822d..f185861 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -18,11 +18,22 @@ # Source this file from the $IMPALA_HOME directory to # setup your environment. If $IMPALA_HOME is undefined # this script will set it to the current working directory. - +# +# Some config variables can be overridden. All overridable variables can be overridden +# by impala-config-branch.sh, which in turn can be by impala-config-local.sh. Some config +# variables in the second part of this file (e.g. locations of dependencies, secret keys) +# can be also overridden by setting environment variables before sourcing this file. We +# don't support this for variables that change between branches and versions, e.g. +# version numbers because it creates a "sticky config variable" problem where an old +# value stays in effect when switching between branches or rebasing until the developer +# opens a new shell. We also do not support overriding of some variables that are +# computed based on the values of other variables. +# # This file must be kept compatible with bash options "set -euo pipefail". Those options # will be set by other scripts before sourcing this file. Those options are not set in # this script because scripts outside this repository may need to be updated and that # is not practical at this time. + export JAVA_HOME="${JAVA_HOME:-/usr/java/default}" if [ ! -d "$JAVA_HOME" ]; then echo "JAVA_HOME must be set to the location of your JDK!" @@ -42,38 +53,117 @@ if [ -z "$IMPALA_HOME" ]; then fi fi -: ${IMPALA_TOOLCHAIN="$IMPALA_HOME/toolchain"} +export IMPALA_TOOLCHAIN=${IMPALA_TOOLCHAIN-"$IMPALA_HOME/toolchain"} if [ -z "$IMPALA_TOOLCHAIN" ]; then echo "IMPALA_TOOLCHAIN must be specified. Please set it to a valid directory or"\ "leave it unset." return 1 fi -# If true, will not call $IMPALA_HOME/bin/bootstrap_toolchain.py. -: ${SKIP_TOOLCHAIN_BOOTSTRAP=false} +####################################################################################### +# Variables that can be overridden by impala-config-*.sh but not by environment vars. # +# All component versions and other variables that get updated periodically or between # +# branches go here to avoid the "sticky variable" problem (IMPALA-4653) where the # +# variable from a previously-sourced impala-config.sh overrides the new value. # +####################################################################################### # The unique build id of the toolchain to use if bootstrapping. This is generated by the # native-toolchain build when publishing its build artifacts. This should be changed when # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -: ${IMPALA_TOOLCHAIN_BUILD_ID=308-96a4cc516e} +export IMPALA_TOOLCHAIN_BUILD_ID=308-96a4cc516e + +# Versions of toolchain dependencies. +# ----------------------------------- +export IMPALA_AVRO_VERSION=1.7.4-p4 +export IMPALA_BINUTILS_VERSION=2.26-p1 +export IMPALA_BOOST_VERSION=1.57.0 +export IMPALA_BREAKPAD_VERSION=20150612-p1 +export IMPALA_BZIP2_VERSION=1.0.6-p2 +export IMPALA_CMAKE_VERSION=3.2.3-p1 +export IMPALA_CYRUS_SASL_VERSION=2.1.23 +export IMPALA_GCC_VERSION=4.9.2 +export IMPALA_GFLAGS_VERSION=2.0 +export IMPALA_GLOG_VERSION=0.3.2-p2 +export IMPALA_GPERFTOOLS_VERSION=2.5 +export IMPALA_GTEST_VERSION=1.6.0 +export IMPALA_LLVM_VERSION=3.8.0-p1 +export IMPALA_LLVM_ASAN_VERSION=3.8.0-p1 +# Debug builds should use the release+asserts build to get additional coverage. +# Don't use the LLVM debug build because the binaries are too large to distribute. +export IMPALA_LLVM_DEBUG_VERSION=3.8.0-asserts-p1 +export IMPALA_LZ4_VERSION=svn +export IMPALA_OPENLDAP_VERSION=2.4.25 +export IMPALA_OPENSSL_VERSION=0.9.8zf +export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=9.0-801 +export IMPALA_RAPIDJSON_VERSION=0.11 +export IMPALA_RE2_VERSION=20130115-p1 +export IMPALA_SNAPPY_VERSION=1.1.3 +export IMPALA_SQUEASEL_VERSION=3.3 +# TPC utilities used for test/benchmark data generation. +export IMPALA_TPC_DS_VERSION=2.1.0 +export IMPALA_TPC_H_VERSION=2.17.0 +export IMPALA_THRIFT_VERSION=0.9.0-p8 +export IMPALA_THRIFT_JAVA_VERSION=0.9.0 +export IMPALA_ZLIB_VERSION=1.2.8 + +if [[ $OSTYPE == "darwin"* ]]; then + IMPALA_CYRUS_SASL_VERSION=2.1.26 + IMPALA_GPERFTOOLS_VERSION=2.3 + IMPALA_OPENSSL_VERSION=1.0.1p + IMPALA_THRIFT_VERSION=0.9.2 + IMPALA_THRIFT_JAVA_VERSION=0.9.2 +fi + +# Kudu version in the toolchain; provides libkudu_client.so and minicluster binaries. +export IMPALA_KUDU_VERSION=e018a83 + +# Kudu version used to identify Java client jar from maven +export KUDU_JAVA_VERSION=1.2.0-SNAPSHOT + +# Versions of Hadoop ecosystem dependencies. +# ------------------------------------------ +export CDH_MAJOR_VERSION=5 +export IMPALA_HADOOP_VERSION=2.6.0-cdh5.11.0-SNAPSHOT +export IMPALA_HBASE_VERSION=1.2.0-cdh5.11.0-SNAPSHOT +export IMPALA_HIVE_VERSION=1.1.0-cdh5.11.0-SNAPSHOT +export IMPALA_SENTRY_VERSION=1.5.1-cdh5.11.0-SNAPSHOT +export IMPALA_PARQUET_VERSION=1.5.0-cdh5.11.0-SNAPSHOT +export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0 + +# Source the branch and local config override files here to override any +# variables above or any variables below that allow overriding via environment +# variable. +. "$IMPALA_HOME/bin/impala-config-branch.sh" +if [ -f "$IMPALA_HOME/bin/impala-config-local.sh" ]; then + . "$IMPALA_HOME/bin/impala-config-local.sh" +fi + +######################################################################################### +# Below here are variables that can be overridden by impala-config-*.sh and environment # +# vars, variables computed based on other variables, and variables that cannot be # +# overridden. # +######################################################################################### + +# If true, will not call $IMPALA_HOME/bin/bootstrap_toolchain.py. +export SKIP_TOOLCHAIN_BOOTSTRAP=${SKIP_TOOLCHAIN_BOOTSTRAP-false} # This flag is used in $IMPALA_HOME/cmake_modules/toolchain.cmake. # If it's 0, Impala will be built with the compiler in the toolchain directory. -: ${USE_SYSTEM_GCC=0} +export USE_SYSTEM_GCC=${USE_SYSTEM_GCC-0} # Use ld.gold instead of ld by default to speed up builds. -: ${USE_GOLD_LINKER=true} +export USE_GOLD_LINKER=${USE_GOLD_LINKER-true} # Override the default compiler by setting a path to the new compiler. The default # compiler depends on USE_SYSTEM_GCC and IMPALA_GCC_VERSION. The intended use case # is to set the compiler to distcc, in that case the user would also set # IMPALA_BUILD_THREADS to increase parallelism. -: ${IMPALA_CXX_COMPILER=default} +export IMPALA_CXX_COMPILER=${IMPALA_CXX_COMPILER-default} # If enabled, debug symbols are added to cross-compiled IR. -: ${ENABLE_IMPALA_IR_DEBUG_INFO=false} +export ENABLE_IMPALA_IR_DEBUG_INFO=${ENABLE_IMPALA_IR_DEBUG_INFO-false} if [ -d "$IMPALA_HOME/thirdparty" ]; then NO_THIRDPARTY=false @@ -82,16 +172,8 @@ else fi # If true, download and use the CDH components from S3 instead of the ones # in $IMPALA_HOME/thirdparty. -: ${DOWNLOAD_CDH_COMPONENTS="$NO_THIRDPARTY"} - -export IMPALA_TOOLCHAIN -export SKIP_TOOLCHAIN_BOOTSTRAP -export IMPALA_TOOLCHAIN_BUILD_ID -export USE_SYSTEM_GCC -export USE_GOLD_LINKER -export IMPALA_CXX_COMPILER -export ENABLE_IMPALA_IR_DEBUG_INFO -export DOWNLOAD_CDH_COMPONENTS +export DOWNLOAD_CDH_COMPONENTS=${DOWNLOAD_CDH_COMPONENTS-"$NO_THIRDPARTY"} + export IS_OSX="$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false; fi)" # To use a local build of Kudu, set KUDU_BUILD_DIR to the path Kudu was built in and @@ -104,10 +186,8 @@ export IS_OSX="$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo fals # cmake <path to Kudu source dir> # make # DESTDIR=$KUDU_CLIENT_DIR make install -: ${KUDU_BUILD_DIR=} -: ${KUDU_CLIENT_DIR=} -export KUDU_BUILD_DIR -export KUDU_CLIENT_DIR +export KUDU_BUILD_DIR=${KUDU_BUILD_DIR-} +export KUDU_CLIENT_DIR=${KUDU_CLIENT_DIR-} if [[ -n "$KUDU_BUILD_DIR" && -z "$KUDU_CLIENT_DIR" ]]; then echo When KUDU_BUILD_DIR is set KUDU_CLIENT_DIR must also be set. 1>&2 return 1 @@ -117,8 +197,8 @@ if [[ -z "$KUDU_BUILD_DIR" && -n "$KUDU_CLIENT_DIR" ]]; then return 1 fi -: ${USE_KUDU_DEBUG_BUILD=false} # Only applies when using Kudu from the toolchain -export USE_KUDU_DEBUG_BUILD +# Only applies when using Kudu from the toolchain +export USE_KUDU_DEBUG_BUILD=${USE_KUDU_DEBUG_BUILD-false} # Kudu doesn't compile on some old Linux distros. KUDU_IS_SUPPORTED enables building Kudu # into the backend. The frontend build is OS independent since it is Java. @@ -149,7 +229,6 @@ if [[ -z "${KUDU_IS_SUPPORTED-}" ]]; then fi export KUDU_IS_SUPPORTED -export CDH_MAJOR_VERSION=5 export HADOOP_LZO="${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo}" export IMPALA_LZO="${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo}" export IMPALA_AUX_TEST_HOME="${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests}" @@ -244,63 +323,9 @@ else fi export NUM_CONCURRENT_TESTS="${NUM_CONCURRENT_TESTS-${CORES}}" -# Versions of toolchain dependencies. -export IMPALA_AVRO_VERSION=1.7.4-p4 -export IMPALA_BINUTILS_VERSION=2.26-p1 -export IMPALA_BOOST_VERSION=1.57.0 -export IMPALA_BREAKPAD_VERSION=20150612-p1 -export IMPALA_BZIP2_VERSION=1.0.6-p2 -export IMPALA_CMAKE_VERSION=3.2.3-p1 -export IMPALA_CYRUS_SASL_VERSION=2.1.23 -export IMPALA_GCC_VERSION=4.9.2 -export IMPALA_GFLAGS_VERSION=2.0 -export IMPALA_GLOG_VERSION=0.3.2-p2 -export IMPALA_GPERFTOOLS_VERSION=2.5 -export IMPALA_GTEST_VERSION=1.6.0 -export IMPALA_LLVM_VERSION=3.8.0-p1 -export IMPALA_LLVM_ASAN_VERSION=3.8.0-p1 -# Debug builds should use the release+asserts build to get additional coverage. -# Don't use the LLVM debug build because the binaries are too large to distribute. -export IMPALA_LLVM_DEBUG_VERSION=3.8.0-asserts-p1 -export IMPALA_LZ4_VERSION=svn -export IMPALA_OPENLDAP_VERSION=2.4.25 -export IMPALA_OPENSSL_VERSION=0.9.8zf -export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=9.0-801 -export IMPALA_RAPIDJSON_VERSION=0.11 -export IMPALA_RE2_VERSION=20130115-p1 -export IMPALA_SNAPPY_VERSION=1.1.3 -export IMPALA_SQUEASEL_VERSION=3.3 -# TPC utilities used for test/benchmark data generation. -export IMPALA_TPC_DS_VERSION=2.1.0 -export IMPALA_TPC_H_VERSION=2.17.0 -export IMPALA_THRIFT_VERSION=0.9.0-p8 -export IMPALA_THRIFT_JAVA_VERSION=0.9.0 -export IMPALA_ZLIB_VERSION=1.2.8 - -# Kudu version in the toolchain; provides libkudu_client.so and minicluster binaries. -export IMPALA_KUDU_VERSION=e018a83 - -# Kudu version used to identify Java client jar from maven -export KUDU_JAVA_VERSION=1.2.0-SNAPSHOT - export KUDU_MASTER="${KUDU_MASTER:-127.0.0.1}" export KUDU_MASTER_PORT="${KUDU_MASTER_PORT:-7051}" -if [[ $OSTYPE == "darwin"* ]]; then - IMPALA_CYRUS_SASL_VERSION=2.1.26 - IMPALA_GPERFTOOLS_VERSION=2.3 - IMPALA_OPENSSL_VERSION=1.0.1p - IMPALA_THRIFT_VERSION=0.9.2 - IMPALA_THRIFT_JAVA_VERSION=0.9.2 -fi - -export IMPALA_HADOOP_VERSION=${IMPALA_HADOOP_VERSION:-2.6.0-cdh5.11.0-SNAPSHOT} -export IMPALA_HBASE_VERSION=${IMPALA_HBASE_VERSION:-1.2.0-cdh5.11.0-SNAPSHOT} -export IMPALA_HIVE_VERSION=${IMPALA_HIVE_VERSION:-1.1.0-cdh5.11.0-SNAPSHOT} -export IMPALA_SENTRY_VERSION=${IMPALA_SENTRY_VERSION:-1.5.1-cdh5.11.0-SNAPSHOT} -export IMPALA_PARQUET_VERSION=${IMPALA_PARQUET_VERSION:-1.5.0-cdh5.11.0-SNAPSHOT} -export IMPALA_LLAMA_MINIKDC_VERSION=${IMPALA_LLAMA_MINIKDC_VERSION:-1.0.0} - export IMPALA_FE_DIR="$IMPALA_HOME/fe" export IMPALA_BE_DIR="$IMPALA_HOME/be" export IMPALA_WORKLOAD_DIR="$IMPALA_HOME/testdata/workloads" @@ -326,10 +351,9 @@ export HADOOP_CONF_DIR="$IMPALA_FE_DIR/src/test/resources" export HADOOP_INCLUDE_DIR=${HADOOP_INCLUDE_DIR:-"${HADOOP_HOME}/include"} export HADOOP_LIB_DIR=${HADOOP_LIB_DIR:-"${HADOOP_HOME}/lib"} -: ${HADOOP_CLASSPATH=} # Please note that the * is inside quotes, thus it won't get expanded by bash but # by java, see "Understanding class path wildcards" at http://goo.gl/f0cfft -export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HADOOP_HOME}/share/hadoop/tools/lib/*" +export HADOOP_CLASSPATH="${HADOOP_CLASSPATH-}:${HADOOP_HOME}/share/hadoop/tools/lib/*" # YARN is configured to use LZO so the LZO jar needs to be in the hadoop classpath. export LZO_JAR_PATH="$HADOOP_LZO/build/hadoop-lzo-0.4.15.jar" HADOOP_CLASSPATH+=":$LZO_JAR_PATH" @@ -380,12 +404,10 @@ export ASAN_SYMBOLIZER_PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION export CLUSTER_DIR="${IMPALA_HOME}/testdata/cluster" # The number of parallel build processes we should run at a time. -: ${IMPALA_BUILD_THREADS:="$(nproc)"} -export IMPALA_BUILD_THREADS +export IMPALA_BUILD_THREADS=${IMPALA_BUILD_THREADS-"$(nproc)"} # Additional flags to pass to make or ninja. -: ${IMPALA_MAKE_FLAGS:=""} -export IMPALA_MAKE_FLAGS +export IMPALA_MAKE_FLAGS=${IMPALA_MAKE_FLAGS-} # Some environments (like the packaging build) might not have $USER set. Fix that here. export USER="${USER-`id -un`}" @@ -401,12 +423,11 @@ export USER="${USER-`id -un`}" # TODO: figure out how to turn this off only the stuff that can't run with it. #LIBHDFS_OPTS="-Xcheck:jni -Xcheck:nabounds" # - Points to the location of libbackend.so. -LIBHDFS_OPTS="${LIBHDFS_OPTS:-}" -LIBHDFS_OPTS="${LIBHDFS_OPTS} -Djava.library.path=${HADOOP_LIB_DIR}/native/" +export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${HADOOP_LIB_DIR}/native/" # READER BEWARE: This always points to the debug build. # TODO: Consider having cmake scripts change this value depending on # the build type. -export LIBHDFS_OPTS="${LIBHDFS_OPTS}:${IMPALA_HOME}/be/build/debug/service" +LIBHDFS_OPTS="${LIBHDFS_OPTS}:${IMPALA_HOME}/be/build/debug/service" export ARTISTIC_STYLE_OPTIONS="$IMPALA_BE_DIR/.astylerc" @@ -419,8 +440,7 @@ export JAVA_LIBRARY_PATH="${IMPALA_SNAPPY_PATH}" LIB_JAVA=`find "${JAVA_HOME}/" -name libjava.so | head -1` LIB_JSIG=`find "${JAVA_HOME}/" -name libjsig.so | head -1` LIB_JVM=` find "${JAVA_HOME}/" -name libjvm.so | head -1` -LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}" -LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JVM}`" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HADOOP_LIB_DIR}/native" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_HOME}/be/build/debug/service" @@ -432,16 +452,12 @@ if [ $USE_SYSTEM_GCC -eq 0 ]; then LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN_GCC_LIB}" fi -export LD_LIBRARY_PATH -LD_PRELOAD="${LD_PRELOAD-}" -export LD_PRELOAD="${LD_PRELOAD}:${LIB_JSIG}" +export LD_PRELOAD="${LD_PRELOAD-}:${LIB_JSIG}" -CLASSPATH="${CLASSPATH-}" -CLASSPATH="$IMPALA_FE_DIR/target/dependency:$CLASSPATH" +export CLASSPATH="$IMPALA_FE_DIR/target/dependency:${CLASSPATH-}" CLASSPATH="$IMPALA_FE_DIR/target/classes:$CLASSPATH" CLASSPATH="$IMPALA_FE_DIR/src/test/resources:$CLASSPATH" CLASSPATH="$LZO_JAR_PATH:$CLASSPATH" -export CLASSPATH # Setup aliases # Helper alias to script that verifies and merges Gerrit changes
