IMPALA-4006: dangerous rm -rf statements in scripts Quoted variable substitutions in rm -rf commands and in many other places. This prevents disasters if those variables contain whitespace.
Redirected output of the cd commands to /dev/null. This prevents polluting the target variable with the directory name when the CDPATH environment variable is set. Change-Id: I7503794180dee99eeb979e67f34e3b2edade70fe Reviewed-on: http://gerrit.cloudera.org:8080/4078 Tested-by: Internal Jenkins Reviewed-by: Tim Armstrong <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a60ba6d2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a60ba6d2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a60ba6d2 Branch: refs/heads/master Commit: a60ba6d2743eee5f2e5c5c3f1baa375ede0195e5 Parents: 2cd7524 Author: Zoltan Ivanfi <[email protected]> Authored: Mon Aug 22 17:29:44 2016 +0200 Committer: Tim Armstrong <[email protected]> Committed: Thu Sep 1 21:26:52 2016 +0000 ---------------------------------------------------------------------- bin/clean.sh | 20 +-- bin/impala-config.sh | 200 ++++++++++++++------------- bin/impala-python | 2 +- bin/impala-python-common.sh | 4 +- bin/run-all-tests.sh | 44 +++--- bin/set-classpath.sh | 26 ++-- buildall.sh | 104 +++++++------- infra/python/deps/download_requirements | 12 +- 8 files changed, 209 insertions(+), 203 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/clean.sh ---------------------------------------------------------------------- diff --git a/bin/clean.sh b/bin/clean.sh index 00d02b3..9ac73cf 100755 --- a/bin/clean.sh +++ b/bin/clean.sh @@ -29,8 +29,8 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0) make clean || : # Stop the minikdc if needed. -if ${CLUSTER_DIR}/admin is_kerberized; then - ${IMPALA_HOME}/testdata/bin/minikdc.sh stop +if "${CLUSTER_DIR}/admin" is_kerberized; then + "${IMPALA_HOME}/testdata/bin/minikdc.sh" stop fi # clean the external data source project @@ -45,35 +45,35 @@ pushd $IMPALA_FE_DIR rm -rf target rm -f src/test/resources/{core,hbase,hive}-site.xml rm -rf generated-sources/* -rm -rf ${IMPALA_LOGS_DIR}/* +[ -z "$IMPALA_LOGS_DIR" ] || rm -rf "${IMPALA_LOGS_DIR}"/* mkdir -p $IMPALA_ALL_LOGS_DIRS popd # clean be -pushd $IMPALA_HOME/be +pushd "$IMPALA_HOME/be" # remove everything listed in .gitignore git clean -Xdfq popd # clean shell build artifacts -pushd $IMPALA_HOME/shell +pushd "$IMPALA_HOME/shell" # remove everything listed in .gitignore git clean -Xdfq popd # Clean stale .pyc, .pyo files and __pycache__ directories. -pushd ${IMPALA_HOME} +pushd "${IMPALA_HOME}" find . -type f -name "*.py[co]" -delete find . -type d -name "__pycache__" -delete popd # clean llvm -rm -f $IMPALA_HOME/llvm-ir/impala*.ll -rm -f $IMPALA_HOME/be/generated-sources/impala-ir/* +rm -f "$IMPALA_HOME/llvm-ir/"impala*.ll +rm -f "$IMPALA_HOME/be/generated-sources/impala-ir/"* # Cleanup Impala-lzo -if [ -e $IMPALA_LZO ]; then - pushd $IMPALA_LZO; git clean -fdx .; popd +if [ -e "$IMPALA_LZO" ]; then + pushd "$IMPALA_LZO"; git clean -fdx .; popd fi # When switching to and from toolchain, make sure to remove all CMake generated files http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index f79d8f8..7f4df7f 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -34,16 +34,16 @@ if [[ ! -e "$JAVA" ]]; then return 1 fi -if [ -z $IMPALA_HOME ]; then - if [[ ! -z $ZSH_NAME ]]; then - export IMPALA_HOME=$(dirname $(cd $(dirname ${(%):-%x}) && pwd)) +if [ -z "$IMPALA_HOME" ]; then + if [[ ! -z "$ZSH_NAME" ]]; then + export IMPALA_HOME=$(dirname "$(cd $(dirname ${(%):-%x}) >/dev/null && pwd)") else - export IMPALA_HOME=$(dirname $(cd $(dirname "${BASH_SOURCE[0]}") && pwd)) + export IMPALA_HOME=$(dirname "$(cd $(dirname "${BASH_SOURCE[0]}") >/dev/null && pwd)") fi fi -: ${IMPALA_TOOLCHAIN=$IMPALA_HOME/toolchain} -if [ -z $IMPALA_TOOLCHAIN ]; then +: ${IMPALA_TOOLCHAIN="$IMPALA_HOME/toolchain"} +if [ -z "$IMPALA_TOOLCHAIN" ]; then echo "IMPALA_TOOLCHAIN must be specified. Please set it to a valid directory or"\ "leave it unset." return 1 @@ -69,14 +69,14 @@ fi # If enabled, debug symbols are added to cross-compiled IR. : ${ENABLE_IMPALA_IR_DEBUG_INFO=false} -if [ -d $IMPALA_HOME/thirdparty ]; then +if [ -d "$IMPALA_HOME/thirdparty" ]; then NO_THIRDPARTY=false else NO_THIRDPARTY=true fi # If true, download and use the CDH components from S3 instead of the ones # in $IMPALA_HOME/thirdparty. -: ${DOWNLOAD_CDH_COMPONENTS=$NO_THIRDPARTY} +: ${DOWNLOAD_CDH_COMPONENTS="$NO_THIRDPARTY"} export IMPALA_TOOLCHAIN export SKIP_TOOLCHAIN_BOOTSTRAP @@ -85,7 +85,7 @@ export USE_GOLD_LINKER export IMPALA_CXX_COMPILER export ENABLE_IMPALA_IR_DEBUG_INFO export DOWNLOAD_CDH_COMPONENTS -export IS_OSX=$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false; fi) +export IS_OSX="$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false; fi)" # To use a local build of Kudu, set KUDU_BUILD_DIR to the path Kudu was built in and # set KUDU_CLIENT_DIR to the path KUDU was installed in. @@ -101,11 +101,11 @@ export IS_OSX=$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false : ${KUDU_CLIENT_DIR=} export KUDU_BUILD_DIR export KUDU_CLIENT_DIR -if [[ -n $KUDU_BUILD_DIR && -z $KUDU_CLIENT_DIR ]]; then +if [[ -n "$KUDU_BUILD_DIR" && -z "$KUDU_CLIENT_DIR" ]]; then echo When KUDU_BUILD_DIR is set KUDU_CLIENT_DIR must also be set. 1>&2 return 1 fi -if [[ -z $KUDU_BUILD_DIR && -n $KUDU_CLIENT_DIR ]]; then +if [[ -z "$KUDU_BUILD_DIR" && -n "$KUDU_CLIENT_DIR" ]]; then echo When KUDU_CLIENT_DIR is set KUDU_BUILD_DIR must also be set. 1>&2 return 1 fi @@ -117,20 +117,20 @@ export USE_KUDU_DEBUG_BUILD # into the backend. The frontend build is OS independent since it is Java. if [[ -z "${KUDU_IS_SUPPORTED-}" ]]; then KUDU_IS_SUPPORTED=true - if [[ -z $KUDU_BUILD_DIR ]]; then + if [[ -z "$KUDU_BUILD_DIR" ]]; then if ! $IS_OSX; then if ! which lsb_release &>/dev/null; then echo Unable to find the 'lsb_release' command. \ Please ensure it is available in your PATH. 1>&2 return 1 fi - DISTRO_VERSION=$(lsb_release -sir 2>&1) + DISTRO_VERSION="$(lsb_release -sir 2>&1)" if [[ $? -ne 0 ]]; then echo lsb_release cammond failed, output was: "$DISTRO_VERSION" 1>&2 return 1 fi # Remove spaces, trim minor versions, and convert to lowercase. - DISTRO_VERSION=$(tr -d ' \n' <<< "$DISTRO_VERSION" | cut -d. -f1 | tr "A-Z" "a-z") + DISTRO_VERSION="$(tr -d ' \n' <<< "$DISTRO_VERSION" | cut -d. -f1 | tr "A-Z" "a-z")" case "$DISTRO_VERSION" in # "enterprise" is Oracle centos5 | debian* | enterprise*5 | redhat*5 | suse* | ubuntu*12) @@ -142,18 +142,18 @@ fi export KUDU_IS_SUPPORTED export CDH_MAJOR_VERSION=5 -export HADOOP_LZO=${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo} -export IMPALA_LZO=${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo} -export IMPALA_AUX_TEST_HOME=${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests} -export TARGET_FILESYSTEM=${TARGET_FILESYSTEM-"hdfs"} -export FILESYSTEM_PREFIX=${FILESYSTEM_PREFIX-""} -export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-"DummySecretAccessKey"} -export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-"DummyAccessKeyId"} -export S3_BUCKET=${S3_BUCKET-""} -export HDFS_REPLICATION=${HDFS_REPLICATION-3} -export ISILON_NAMENODE=${ISILON_NAMENODE-""} -export DEFAULT_FS=${DEFAULT_FS-"hdfs://localhost:20500"} -export WAREHOUSE_LOCATION_PREFIX=${WAREHOUSE_LOCATION_PREFIX-""} +export HADOOP_LZO="${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo}" +export IMPALA_LZO="${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo}" +export IMPALA_AUX_TEST_HOME="${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests}" +export TARGET_FILESYSTEM="${TARGET_FILESYSTEM-hdfs}" +export FILESYSTEM_PREFIX="${FILESYSTEM_PREFIX-}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY-DummySecretAccessKey}" +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID-DummyAccessKeyId}" +export S3_BUCKET="${S3_BUCKET-}" +export HDFS_REPLICATION="${HDFS_REPLICATION-3}" +export ISILON_NAMENODE="${ISILON_NAMENODE-}" +export DEFAULT_FS="${DEFAULT_FS-hdfs://localhost:20500}" +export WAREHOUSE_LOCATION_PREFIX="${WAREHOUSE_LOCATION_PREFIX-}" export LOCAL_FS="file:${WAREHOUSE_LOCATION_PREFIX}" export METASTORE_DB="hive_impala" @@ -198,8 +198,8 @@ elif [ "${TARGET_FILESYSTEM}" = "local" ]; then "'$WAREHOUSE_LOCATION_PREFIX'" return 1 fi - export DEFAULT_FS=${LOCAL_FS} - export FILESYSTEM_PREFIX=${LOCAL_FS} + export DEFAULT_FS="${LOCAL_FS}" + export FILESYSTEM_PREFIX="${LOCAL_FS}" elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then echo "Unsupported filesystem '$TARGET_FILESYSTEM'" echo "Valid values are: hdfs, isilon, s3, local" @@ -207,15 +207,15 @@ elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then fi # Directories where local cluster logs will go when running tests or loading data -export IMPALA_LOGS_DIR=${IMPALA_HOME}/logs -export IMPALA_CLUSTER_LOGS_DIR=${IMPALA_LOGS_DIR}/cluster -export IMPALA_DATA_LOADING_LOGS_DIR=${IMPALA_LOGS_DIR}/data_loading -export IMPALA_DATA_LOADING_SQL_DIR=${IMPALA_DATA_LOADING_LOGS_DIR}/sql -export IMPALA_FE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/fe_tests -export IMPALA_FE_TEST_COVERAGE_DIR=${IMPALA_FE_TEST_LOGS_DIR}/coverage -export IMPALA_BE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/be_tests -export IMPALA_EE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/ee_tests -export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/custom_cluster_tests +export IMPALA_LOGS_DIR="${IMPALA_HOME}/logs" +export IMPALA_CLUSTER_LOGS_DIR="${IMPALA_LOGS_DIR}/cluster" +export IMPALA_DATA_LOADING_LOGS_DIR="${IMPALA_LOGS_DIR}/data_loading" +export IMPALA_DATA_LOADING_SQL_DIR="${IMPALA_DATA_LOADING_LOGS_DIR}/sql" +export IMPALA_FE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/fe_tests" +export IMPALA_FE_TEST_COVERAGE_DIR="${IMPALA_FE_TEST_LOGS_DIR}/coverage" +export IMPALA_BE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/be_tests" +export IMPALA_EE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/ee_tests" +export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/custom_cluster_tests" # List of all Impala log dirs and create them. export IMPALA_ALL_LOGS_DIRS="${IMPALA_CLUSTER_LOGS_DIR} ${IMPALA_DATA_LOADING_LOGS_DIR} ${IMPALA_DATA_LOADING_SQL_DIR} @@ -229,21 +229,21 @@ mkdir -p $IMPALA_ALL_LOGS_DIRS # hardwired in its code, so we cannot change the output dir by configuration. # We create two symlinks to capture the logs when running ctest either from # ${IMPALA_HOME} or ${IMPALA_HOME}/be. -rm -rf ${IMPALA_HOME}/Testing -mkdir -p ${IMPALA_HOME}/Testing -ln -fs ${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_HOME}/Testing/Temporary -rm -rf ${IMPALA_HOME}/be/Testing -mkdir -p ${IMPALA_HOME}/be/Testing -ln -fs ${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_HOME}/be/Testing/Temporary +rm -rf "${IMPALA_HOME}/Testing" +mkdir -p "${IMPALA_HOME}/Testing" +ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/Testing/Temporary" +rm -rf "${IMPALA_HOME}/be/Testing" +mkdir -p "${IMPALA_HOME}/be/Testing" +ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/be/Testing/Temporary" # Reduce the concurrency for local tests to half the number of cores in the system. # Note than nproc may not be available on older distributions (centos5.5) if type nproc >/dev/null 2>&1; then CORES=$(($(nproc) / 2)) else - CORES='4' + CORES=4 fi -export NUM_CONCURRENT_TESTS=${NUM_CONCURRENT_TESTS-${CORES}} +export NUM_CONCURRENT_TESTS="${NUM_CONCURRENT_TESTS-${CORES}}" # Versions of toolchain dependencies (or if toolchain is not used of dependencies in # thirdparty) @@ -280,8 +280,8 @@ export IMPALA_THRIFT_VERSION=0.9.0-p8 export IMPALA_THRIFT_JAVA_VERSION=0.9.0 export IMPALA_ZLIB_VERSION=1.2.8 -export KUDU_MASTER=${KUDU_MASTER:-"127.0.0.1"} -export KUDU_MASTER_PORT=${KUDU_MASTER_PORT:-"7051"} +export KUDU_MASTER="${KUDU_MASTER:-127.0.0.1}" +export KUDU_MASTER_PORT="${KUDU_MASTER_PORT:-7051}" # TODO: Figure out a way to use a snapshot version without causing a lot of breakage due # to nightly changes from Kudu. The version below is the last released version but # before release this needs to be updated to the version about to be released. @@ -303,83 +303,85 @@ export IMPALA_LLAMA_VERSION=1.0.0-cdh5.10.0-SNAPSHOT export IMPALA_PARQUET_VERSION=1.5.0-cdh5.10.0-SNAPSHOT export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0 -export IMPALA_FE_DIR=$IMPALA_HOME/fe -export IMPALA_BE_DIR=$IMPALA_HOME/be -export IMPALA_WORKLOAD_DIR=$IMPALA_HOME/testdata/workloads -export IMPALA_AUX_WORKLOAD_DIR=$IMPALA_AUX_TEST_HOME/testdata/workloads -export IMPALA_DATASET_DIR=$IMPALA_HOME/testdata/datasets -export IMPALA_AUX_DATASET_DIR=$IMPALA_AUX_TEST_HOME/testdata/datasets -export IMPALA_COMMON_DIR=$IMPALA_HOME/common -export PATH=$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH +export IMPALA_FE_DIR="$IMPALA_HOME/fe" +export IMPALA_BE_DIR="$IMPALA_HOME/be" +export IMPALA_WORKLOAD_DIR="$IMPALA_HOME/testdata/workloads" +export IMPALA_AUX_WORKLOAD_DIR="$IMPALA_AUX_TEST_HOME/testdata/workloads" +export IMPALA_DATASET_DIR="$IMPALA_HOME/testdata/datasets" +export IMPALA_AUX_DATASET_DIR="$IMPALA_AUX_TEST_HOME/testdata/datasets" +export IMPALA_COMMON_DIR="$IMPALA_HOME/common" +export PATH="$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH" # The directory in which all the thirdparty CDH components live. if [ "${DOWNLOAD_CDH_COMPONENTS}" = true ]; then - export CDH_COMPONENTS_HOME=$IMPALA_TOOLCHAIN/cdh_components + export CDH_COMPONENTS_HOME="$IMPALA_TOOLCHAIN/cdh_components" else - export CDH_COMPONENTS_HOME=$IMPALA_HOME/thirdparty + export CDH_COMPONENTS_HOME="$IMPALA_HOME/thirdparty" fi # Hadoop dependencies are snapshots in the Impala tree -export HADOOP_HOME=$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/ -export HADOOP_CONF_DIR=$IMPALA_FE_DIR/src/test/resources +export HADOOP_HOME="$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/" +export HADOOP_CONF_DIR="$IMPALA_FE_DIR/src/test/resources" : ${HADOOP_CLASSPATH=} -export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:"${HADOOP_HOME}/share/hadoop/tools/lib/*" +# Please note that the * is inside quotes, thus it won't get exanded by bash but +# by java, see "Understanding class path wildcards" at http://goo.gl/f0cfft +export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HADOOP_HOME}/share/hadoop/tools/lib/*" # YARN is configured to use LZO so the LZO jar needs to be in the hadoop classpath. export LZO_JAR_PATH="$HADOOP_LZO/build/hadoop-lzo-0.4.15.jar" HADOOP_CLASSPATH+=":$LZO_JAR_PATH" -export MINI_DFS_BASE_DATA_DIR=$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data -export PATH=$HADOOP_HOME/bin:$PATH +export MINI_DFS_BASE_DATA_DIR="$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data" +export PATH="$HADOOP_HOME/bin:$PATH" -export LLAMA_HOME=$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/ -export MINIKDC_HOME=$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION} -export SENTRY_HOME=$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION} -export SENTRY_CONF_DIR=$IMPALA_HOME/fe/src/test/resources +export LLAMA_HOME="$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/" +export MINIKDC_HOME="$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION}" +export SENTRY_HOME="$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}" +export SENTRY_CONF_DIR="$IMPALA_HOME/fe/src/test/resources" -export HIVE_HOME=$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/ -export PATH=$HIVE_HOME/bin:$PATH -export HIVE_CONF_DIR=$IMPALA_FE_DIR/src/test/resources +export HIVE_HOME="$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/" +export PATH="$HIVE_HOME/bin:$PATH" +export HIVE_CONF_DIR="$IMPALA_FE_DIR/src/test/resources" # Hive looks for jar files in a single directory from HIVE_AUX_JARS_PATH plus # any jars in AUX_CLASSPATH. (Or a list of jars in HIVE_AUX_JARS_PATH.) # The Postgres JDBC driver is downloaded by maven when building the frontend. # Export the location of Postgres JDBC driver so Sentry can pick it up. -export POSTGRES_JDBC_DRIVER=${IMPALA_FE_DIR}/target/dependency/postgresql-${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4.jar +export POSTGRES_JDBC_DRIVER="${IMPALA_FE_DIR}/target/dependency/postgresql-${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4.jar" export HIVE_AUX_JARS_PATH="$POSTGRES_JDBC_DRIVER" export AUX_CLASSPATH="${LZO_JAR_PATH}" ### Tell hive not to use jline export HADOOP_USER_CLASSPATH_FIRST=true -export HBASE_HOME=$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/ -export PATH=$HBASE_HOME/bin:$PATH +export HBASE_HOME="$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/" +export PATH="$HBASE_HOME/bin:$PATH" # Add the jars so hive can create hbase tables. -export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-common-${IMPALA_HBASE_VERSION}.jar -export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-client-${IMPALA_HBASE_VERSION}.jar -export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-server-${IMPALA_HBASE_VERSION}.jar -export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-protocol-${IMPALA_HBASE_VERSION}.jar -export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA_HBASE_VERSION}.jar +export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-common-${IMPALA_HBASE_VERSION}.jar" +export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-client-${IMPALA_HBASE_VERSION}.jar" +export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-server-${IMPALA_HBASE_VERSION}.jar" +export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-protocol-${IMPALA_HBASE_VERSION}.jar" +export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA_HBASE_VERSION}.jar" -export HBASE_CONF_DIR=$HIVE_CONF_DIR +export HBASE_CONF_DIR="$HIVE_CONF_DIR" # Set $THRIFT_HOME to the Thrift directory in toolchain. -export THRIFT_HOME=${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION} +export THRIFT_HOME="${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}" # ASAN needs a matching version of llvm-symbolizer to symbolize stack traces. -export ASAN_SYMBOLIZER_PATH=${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer +export ASAN_SYMBOLIZER_PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer" -export CLUSTER_DIR=${IMPALA_HOME}/testdata/cluster +export CLUSTER_DIR="${IMPALA_HOME}/testdata/cluster" -: ${IMPALA_BUILD_THREADS:=$(nproc)} +: ${IMPALA_BUILD_THREADS:="$(nproc)"} export IMPALA_BUILD_THREADS # Some environments (like the packaging build) might not have $USER set. Fix that here. -export USER=${USER-`id -un`} +export USER="${USER-`id -un`}" # Configure python path -. $IMPALA_HOME/bin/set-pythonpath.sh +. "$IMPALA_HOME/bin/set-pythonpath.sh" # These arguments are, despite the name, passed to every JVM created # by an impalad. @@ -396,17 +398,17 @@ LIBHDFS_OPTS="${LIBHDFS_OPTS} -Djava.library.path=${HADOOP_HOME}/lib/native/" # the build type. export LIBHDFS_OPTS="${LIBHDFS_OPTS}:${IMPALA_HOME}/be/build/debug/service" -export ARTISTIC_STYLE_OPTIONS=$IMPALA_BE_DIR/.astylerc +export ARTISTIC_STYLE_OPTIONS="$IMPALA_BE_DIR/.astylerc" -export IMPALA_SNAPPY_PATH=${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib +export IMPALA_SNAPPY_PATH="${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib" -export JAVA_LIBRARY_PATH=${IMPALA_SNAPPY_PATH} +export JAVA_LIBRARY_PATH="${IMPALA_SNAPPY_PATH}" # So that the frontend tests and PlanService can pick up libbackend.so # and other required libraries -LIB_JAVA=`find ${JAVA_HOME}/ -name libjava.so | head -1` -LIB_JSIG=`find ${JAVA_HOME}/ -name libjsig.so | head -1` -LIB_JVM=` find ${JAVA_HOME}/ -name libjvm.so | head -1` +LIB_JAVA=`find "${JAVA_HOME}/" -name libjava.so | head -1` +LIB_JSIG=`find "${JAVA_HOME}/" -name libjsig.so | head -1` +LIB_JVM=` find "${JAVA_HOME}/" -name libjvm.so | head -1` LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JVM}`" @@ -416,7 +418,7 @@ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_SNAPPY_PATH}" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_LZO}/build" if [ $USE_SYSTEM_GCC -eq 0 ]; then - IMPALA_TOOLCHAIN_GCC_LIB=${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64 + IMPALA_TOOLCHAIN_GCC_LIB="${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN_GCC_LIB}" fi @@ -425,10 +427,10 @@ LD_PRELOAD="${LD_PRELOAD-}" export LD_PRELOAD="${LD_PRELOAD}:${LIB_JSIG}" CLASSPATH="${CLASSPATH-}" -CLASSPATH=$IMPALA_FE_DIR/target/dependency:$CLASSPATH -CLASSPATH=$IMPALA_FE_DIR/target/classes:$CLASSPATH -CLASSPATH=$IMPALA_FE_DIR/src/test/resources:$CLASSPATH -CLASSPATH=$LZO_JAR_PATH:$CLASSPATH +CLASSPATH="$IMPALA_FE_DIR/target/dependency:$CLASSPATH" +CLASSPATH="$IMPALA_FE_DIR/target/classes:$CLASSPATH" +CLASSPATH="$IMPALA_FE_DIR/src/test/resources:$CLASSPATH" +CLASSPATH="$LZO_JAR_PATH:$CLASSPATH" export CLASSPATH # Setup aliases @@ -465,9 +467,9 @@ echo "DOWNLOAD_CDH_COMPONENTS = $DOWNLOAD_CDH_COMPONENTS" # work. Note that if impala-config.sh is sourced before the # kerberized cluster is created, it will have to be sourced again # *after* the cluster is created in order to pick up these settings. -export MINIKDC_ENV=${IMPALA_HOME}/testdata/bin/minikdc_env.sh -if ${CLUSTER_DIR}/admin is_kerberized; then - . ${MINIKDC_ENV} +export MINIKDC_ENV="${IMPALA_HOME}/testdata/bin/minikdc_env.sh" +if "${CLUSTER_DIR}/admin" is_kerberized; then + . "${MINIKDC_ENV}" echo " *** This cluster is kerberized ***" echo "KRB5_KTNAME = $KRB5_KTNAME" echo "KRB5_CONFIG = $KRB5_CONFIG" @@ -478,6 +480,6 @@ else # If the cluster *isn't* kerberized, ensure that the environment isn't # polluted with kerberos items that might screw us up. We go through # everything set in the minikdc environment and explicitly unset it. - unset `grep export ${MINIKDC_ENV} | sed "s/.*export \([^=]*\)=.*/\1/" \ + unset `grep export "${MINIKDC_ENV}" | sed "s/.*export \([^=]*\)=.*/\1/" \ | sort | uniq` fi http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-python ---------------------------------------------------------------------- diff --git a/bin/impala-python b/bin/impala-python index 5ff4190..29f9406 100755 --- a/bin/impala-python +++ b/bin/impala-python @@ -1,3 +1,3 @@ #!/bin/bash -source $(dirname "$0")/impala-python-common.sh +source "$(dirname "$0")/impala-python-common.sh" exec "$PY_DIR/env/bin/python" "$@" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-python-common.sh ---------------------------------------------------------------------- diff --git a/bin/impala-python-common.sh b/bin/impala-python-common.sh index 2084429..29a36a2 100644 --- a/bin/impala-python-common.sh +++ b/bin/impala-python-common.sh @@ -21,8 +21,8 @@ set -euo pipefail trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR -LD_LIBRARY_PATH+=":$(python $IMPALA_HOME/infra/python/bootstrap_virtualenv.py \ +LD_LIBRARY_PATH+=":$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \ --print-ld-library-path)" -PY_DIR=$(dirname "$0")/../infra/python +PY_DIR="$(dirname "$0")/../infra/python" python "$PY_DIR/bootstrap_virtualenv.py" http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/run-all-tests.sh ---------------------------------------------------------------------- diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh index 4322efc..7d68216 100755 --- a/bin/run-all-tests.sh +++ b/bin/run-all-tests.sh @@ -24,7 +24,7 @@ set -euo pipefail trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR -. $IMPALA_HOME/bin/set-pythonpath.sh +. "$IMPALA_HOME/bin/set-pythonpath.sh" # Allow picking up strategy from environment : ${EXPLORATION_STRATEGY:=core} @@ -32,9 +32,9 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0) : ${MAX_PYTEST_FAILURES:=10} KERB_ARGS="" -. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 -. ${IMPALA_HOME}/testdata/bin/run-step.sh -if ${CLUSTER_DIR}/admin is_kerberized; then +. "${IMPALA_HOME}/bin/impala-config.sh" > /dev/null 2>&1 +. "${IMPALA_HOME}/testdata/bin/run-step.sh" +if "${CLUSTER_DIR}/admin" is_kerberized; then KERB_ARGS="--use_kerberos" fi @@ -69,10 +69,10 @@ while getopts "e:n:c" OPTION do case "$OPTION" in e) - EXPLORATION_STRATEGY=$OPTARG + EXPLORATION_STRATEGY="$OPTARG" ;; n) - NUM_TEST_ITERATIONS=$OPTARG + NUM_TEST_ITERATIONS="$OPTARG" ;; c) CODE_COVERAGE=true @@ -98,7 +98,7 @@ if [[ "${TARGET_FILESYSTEM}" == "local" ]]; then fi # For logging when using run-step. -LOG_DIR=${IMPALA_EE_TEST_LOGS_DIR} +LOG_DIR="${IMPALA_EE_TEST_LOGS_DIR}" # Enable core dumps ulimit -c unlimited @@ -108,7 +108,7 @@ if [[ "${TARGET_FILESYSTEM}" == "hdfs" ]]; then # script. Restarting HBase will change the region server assignment. Run split-hbase.sh # before running any test. run-step "Split and assign HBase regions" split-hbase.log \ - ${IMPALA_HOME}/testdata/bin/split-hbase.sh + "${IMPALA_HOME}/testdata/bin/split-hbase.sh" fi for i in $(seq 1 $NUM_TEST_ITERATIONS) @@ -116,7 +116,7 @@ do TEST_RET_CODE=0 run-step "Starting Impala cluster" start-impala-cluster.log \ - ${IMPALA_HOME}/bin/start-impala-cluster.py --log_dir=${IMPALA_EE_TEST_LOGS_DIR} \ + "${IMPALA_HOME}/bin/start-impala-cluster.py" --log_dir="${IMPALA_EE_TEST_LOGS_DIR}" \ ${TEST_START_CLUSTER_ARGS} if [[ "$BE_TEST" == true ]]; then @@ -127,14 +127,14 @@ do export SKIP_BE_TEST_PATTERN="session*" fi # Run backend tests. - if ! ${IMPALA_HOME}/bin/run-backend-tests.sh; then + if ! "${IMPALA_HOME}/bin/run-backend-tests.sh"; then TEST_RET_CODE=1 fi fi # Run some queries using run-workload to verify run-workload has not been broken. if ! run-step "Run test run-workload" test-run-workload.log \ - ${IMPALA_HOME}/bin/run-workload.py -w tpch --num_clients=2 --query_names=TPCH-Q1 \ + "${IMPALA_HOME}/bin/run-workload.py" -w tpch --num_clients=2 --query_names=TPCH-Q1 \ --table_format=text/none --exec_options="disable_codegen:False" ${KERB_ARGS}; then TEST_RET_CODE=1 fi @@ -143,7 +143,7 @@ do # Run JUnit frontend tests # Requires a running impalad cluster because some tests (such as DataErrorTest and # JdbcTest) queries against an impala cluster. - pushd ${IMPALA_FE_DIR} + pushd "${IMPALA_FE_DIR}" MVN_ARGS="" if [[ "${TARGET_FILESYSTEM}" == "s3" ]]; then # When running against S3, only run the S3 frontend tests. @@ -152,7 +152,7 @@ do if [[ "$CODE_COVERAGE" == true ]]; then MVN_ARGS+="-DcodeCoverage" fi - if ! ${IMPALA_HOME}/bin/mvn-quiet.sh -fae test ${MVN_ARGS}; then + if ! "${IMPALA_HOME}/bin/mvn-quiet.sh" -fae test ${MVN_ARGS}; then TEST_RET_CODE=1 fi popd @@ -161,7 +161,7 @@ do if [[ "$EE_TEST" == true ]]; then # Run end-to-end tests. # KERBEROS TODO - this will need to deal with ${KERB_ARGS} - if ! ${IMPALA_HOME}/tests/run-tests.py ${COMMON_PYTEST_ARGS} ${EE_TEST_FILES}; then + if ! "${IMPALA_HOME}/tests/run-tests.py" ${COMMON_PYTEST_ARGS} ${EE_TEST_FILES}; then #${KERB_ARGS}; TEST_RET_CODE=1 fi @@ -170,11 +170,11 @@ do if [[ "$JDBC_TEST" == true ]]; then # Run the JDBC tests with background loading disabled. This is interesting because # it requires loading missing table metadata. - ${IMPALA_HOME}/bin/start-impala-cluster.py --log_dir=${IMPALA_EE_TEST_LOGS_DIR} \ + "${IMPALA_HOME}/bin/start-impala-cluster.py" --log_dir="${IMPALA_EE_TEST_LOGS_DIR}" \ --catalogd_args=--load_catalog_in_background=false \ ${TEST_START_CLUSTER_ARGS} - pushd ${IMPALA_FE_DIR} - if ! ${IMPALA_HOME}/bin/mvn-quiet.sh test -Dtest=JdbcTest; then + pushd "${IMPALA_FE_DIR}" + if ! "${IMPALA_HOME}/bin/mvn-quiet.sh" test -Dtest=JdbcTest; then TEST_RET_CODE=1 fi popd @@ -186,17 +186,17 @@ do # pollute the directory with too many files, remove what was there # before. Also, save the IMPALA_MAX_LOG_FILES value for re-set # later. - rm -rf ${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR} - mkdir -p ${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR} - IMPALA_MAX_LOG_FILES_SAVE=${IMPALA_MAX_LOG_FILES:-10} + rm -rf "${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}" + mkdir -p "${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}" + IMPALA_MAX_LOG_FILES_SAVE="${IMPALA_MAX_LOG_FILES:-10}" export IMPALA_MAX_LOG_FILES=0 # Run the custom-cluster tests after all other tests, since they will restart the # cluster repeatedly and lose state. # TODO: Consider moving in to run-tests.py. - if ! ${IMPALA_HOME}/tests/run-custom-cluster-tests.sh ${COMMON_PYTEST_ARGS}; then + if ! "${IMPALA_HOME}/tests/run-custom-cluster-tests.sh" ${COMMON_PYTEST_ARGS}; then TEST_RET_CODE=1 fi - export IMPALA_MAX_LOG_FILES=${IMPALA_MAX_LOG_FILES_SAVE} + export IMPALA_MAX_LOG_FILES="${IMPALA_MAX_LOG_FILES_SAVE}" fi # Finally, run the process failure tests. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/set-classpath.sh ---------------------------------------------------------------------- diff --git a/bin/set-classpath.sh b/bin/set-classpath.sh old mode 100644 new mode 100755 index 873f6ec..68aec2b --- a/bin/set-classpath.sh +++ b/bin/set-classpath.sh @@ -23,20 +23,24 @@ # explicitly to the CLASSPATH. CLASSPATH=\ -$IMPALA_HOME/fe/src/test/resources:\ -$IMPALA_HOME/fe/target/classes:\ -$IMPALA_HOME/fe/target/dependency:\ -$IMPALA_HOME/fe/target/test-classes:\ -${HIVE_HOME}/lib/datanucleus-api-jdo-3.2.1.jar:\ -${HIVE_HOME}/lib/datanucleus-core-3.2.2.jar:\ -${HIVE_HOME}/lib/datanucleus-rdbms-3.2.1.jar: +"$IMPALA_HOME"/fe/src/test/resources:\ +"$IMPALA_HOME"/fe/target/classes:\ +"$IMPALA_HOME"/fe/target/dependency:\ +"$IMPALA_HOME"/fe/target/test-classes:\ +"${HIVE_HOME}"/lib/datanucleus-api-jdo-3.2.1.jar:\ +"${HIVE_HOME}"/lib/datanucleus-core-3.2.2.jar:\ +"${HIVE_HOME}"/lib/datanucleus-rdbms-3.2.1.jar: -for jar in `ls ${IMPALA_HOME}/fe/target/dependency/*.jar`; do - CLASSPATH=${CLASSPATH}:$jar +for jar in "${IMPALA_HOME}"/fe/target/dependency/*.jar; do + if [ -e "$jar" ] ; then + CLASSPATH="${CLASSPATH}:$jar" + fi done -for jar in `ls ${IMPALA_HOME}/testdata/target/dependency/*.jar`; do - CLASSPATH=${CLASSPATH}:$jar +for jar in "${IMPALA_HOME}"/testdata/target/dependency/*.jar; do + if [ -e "$jar" ] ; then + CLASSPATH="${CLASSPATH}:$jar" + fi done export CLASSPATH http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/buildall.sh ---------------------------------------------------------------------- diff --git a/buildall.sh b/buildall.sh index ec79420..1845081 100755 --- a/buildall.sh +++ b/buildall.sh @@ -23,7 +23,7 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0) # run buildall.sh -help to see options ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT=`cd "$ROOT" >/dev/null; pwd` # Grab this *before* we source impala-config.sh to see if the caller has # kerberized environment variables already or not. @@ -33,7 +33,7 @@ if [ ! -z "${MINIKDC_REALM}" ]; then NEEDS_RE_SOURCE_NOTE=0 fi -export IMPALA_HOME=$ROOT +export IMPALA_HOME="$ROOT" if ! . "$ROOT"/bin/impala-config.sh; then echo "Bad configuration, aborting buildall." exit 1 @@ -112,25 +112,25 @@ do EXPLORATION_STRATEGY=exhaustive ;; -snapshot_file) - SNAPSHOT_FILE=${2-} - if [ ! -f $SNAPSHOT_FILE ]; then + SNAPSHOT_FILE="${2-}" + if [ ! -f "$SNAPSHOT_FILE" ]; then echo "-snapshot_file does not exist: $SNAPSHOT_FILE" exit 1; fi TESTDATA_ACTION=1 # Get the full path. - SNAPSHOT_FILE=$(readlink -f $SNAPSHOT_FILE) + SNAPSHOT_FILE="$(readlink -f "$SNAPSHOT_FILE")" shift; ;; -metastore_snapshot_file) - METASTORE_SNAPSHOT_FILE=${2-} - if [ ! -f $METASTORE_SNAPSHOT_FILE ]; then + METASTORE_SNAPSHOT_FILE="${2-}" + if [ ! -f "$METASTORE_SNAPSHOT_FILE" ]; then echo "-metastore_snapshot_file does not exist: $METASTORE_SNAPSHOT_FILE" exit 1; fi TESTDATA_ACTION=1 # Get the full path. - METASTORE_SNAPSHOT_FILE=$(readlink -f $METASTORE_SNAPSHOT_FILE) + METASTORE_SNAPSHOT_FILE="$(readlink -f "$METASTORE_SNAPSHOT_FILE")" shift; ;; -k|-kerberize|-kerberos|-kerb) @@ -236,16 +236,16 @@ if [ ${IMPALA_KERBERIZE} -eq 0 ]; then fi # Loading data on a filesystem other than fs.defaultFS is not supported. -if [[ -z $METASTORE_SNAPSHOT_FILE && "${TARGET_FILESYSTEM}" != "hdfs" && - $TESTDATA_ACTION -eq 1 ]]; then +if [[ -z "$METASTORE_SNAPSHOT_FILE" && "${TARGET_FILESYSTEM}" != "hdfs" && + "$TESTDATA_ACTION" -eq 1 ]]; then echo "The metastore snapshot is required for loading data into ${TARGET_FILESYSTEM}" echo "Use the -metastore_snapshot_file command line paramater." exit 1 fi # option to clean everything first -if [ $CLEAN_ACTION -eq 1 ]; then - $IMPALA_HOME/bin/clean.sh +if [ "$CLEAN_ACTION" -eq 1 ]; then + "$IMPALA_HOME/bin/clean.sh" fi # Populate necessary thirdparty components unless it's set to be skipped. @@ -256,7 +256,7 @@ else echo "Downloading Python dependencies" # Download all the Python dependencies we need before doing anything # of substance. Does not re-download anything that is already present. - if ! $IMPALA_HOME/infra/python/deps/download_requirements; then + if ! "$IMPALA_HOME/infra/python/deps/download_requirements"; then echo "Warning: Unable to download Python requirements." echo "Warning: bootstrap_virtualenv or other Python-based tooling may fail." else @@ -264,88 +264,88 @@ else fi echo "Downloading and extracting toolchain dependencies." - $IMPALA_HOME/bin/bootstrap_toolchain.py + "$IMPALA_HOME/bin/bootstrap_toolchain.py" echo "Toolchain bootstrap complete." fi MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_type=${CMAKE_BUILD_TYPE}" -if [ $BUILD_FE_ONLY -eq 1 ]; then - $IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS} -cmake_only - ${MAKE_CMD} fe +if [ "$BUILD_FE_ONLY" -eq 1 ]; then + "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -cmake_only + "${MAKE_CMD}" fe exit 0 fi -if [ -e $HADOOP_LZO/build/native/Linux-*-*/lib/libgplcompression.so ] +if [ -e "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.so ] then - cp $HADOOP_LZO/build/native/Linux-*-*/lib/libgplcompression.* $HADOOP_HOME/lib/native + cp "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.* "$HADOOP_HOME/lib/native" else echo "No hadoop-lzo found" fi # Stop any running Impala services. -${IMPALA_HOME}/bin/start-impala-cluster.py --kill --force +"${IMPALA_HOME}/bin/start-impala-cluster.py" --kill --force -if [[ $CLEAN_ACTION -eq 1 || $FORMAT_METASTORE -eq 1 || $FORMAT_CLUSTER -eq 1 || - $FORMAT_SENTRY_POLICY_DB -eq 1 || -n $METASTORE_SNAPSHOT_FILE ]] +if [[ "$CLEAN_ACTION" -eq 1 || "$FORMAT_METASTORE" -eq 1 || "$FORMAT_CLUSTER" -eq 1 || + "$FORMAT_SENTRY_POLICY_DB" -eq 1 || -n "$METASTORE_SNAPSHOT_FILE" ]] then # Kill any processes that may be accessing postgres metastore. To be safe, this is done # before we make any changes to the config files. set +e - ${IMPALA_HOME}/testdata/bin/kill-all.sh + "${IMPALA_HOME}/testdata/bin/kill-all.sh" set -e fi CREATE_TEST_CONFIG_ARGS="" -if [[ $FORMAT_SENTRY_POLICY_DB -eq 1 ]]; then +if [[ "$FORMAT_SENTRY_POLICY_DB" -eq 1 ]]; then CREATE_TEST_CONFIG_ARGS+=" -create_sentry_policy_db" fi -if [[ $FORMAT_METASTORE -eq 1 && -z $METASTORE_SNAPSHOT_FILE ]]; then +if [[ "$FORMAT_METASTORE" -eq 1 && -z "$METASTORE_SNAPSHOT_FILE" ]]; then CREATE_TEST_CONFIG_ARGS+=" -create_metastore" fi # Generate the Hadoop configs needed by Impala -${IMPALA_HOME}/bin/create-test-configuration.sh ${CREATE_TEST_CONFIG_ARGS} +"${IMPALA_HOME}/bin/create-test-configuration.sh" ${CREATE_TEST_CONFIG_ARGS} # If a metastore snapshot exists, load it. -if [ $METASTORE_SNAPSHOT_FILE ]; then +if [ "$METASTORE_SNAPSHOT_FILE" ]; then echo "Loading metastore snapshot" - ${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh $METASTORE_SNAPSHOT_FILE + "${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh" "$METASTORE_SNAPSHOT_FILE" fi # build common and backend echo "Calling make_impala.sh ${MAKE_IMPALA_ARGS}" -$IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS} +"$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -if [ -e $IMPALA_LZO ] +if [ -e "$IMPALA_LZO" ] then - pushd $IMPALA_LZO + pushd "$IMPALA_LZO" LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake" rm -f CMakeCache.txt cmake ${LZO_CMAKE_ARGS} - ${MAKE_CMD} + "${MAKE_CMD}" popd fi # build the external data source API -pushd ${IMPALA_HOME}/ext-data-source -${IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests +pushd "${IMPALA_HOME}/ext-data-source" +"${IMPALA_HOME}/bin/mvn-quiet.sh" install -DskipTests popd # build frontend and copy dependencies -pushd ${IMPALA_FE_DIR} -${IMPALA_HOME}/bin/mvn-quiet.sh package -DskipTests +pushd "${IMPALA_FE_DIR}" +"${IMPALA_HOME}/bin/mvn-quiet.sh" package -DskipTests popd # Build the shell tarball echo "Creating shell tarball" -${IMPALA_HOME}/shell/make_shell_tarball.sh +"${IMPALA_HOME}/shell/make_shell_tarball.sh" -if [ $FORMAT_CLUSTER -eq 1 ]; then - $IMPALA_HOME/testdata/bin/run-all.sh -format -elif [ $TESTDATA_ACTION -eq 1 ] || [ $TESTS_ACTION -eq 1 ]; then - $IMPALA_HOME/testdata/bin/run-all.sh +if [ "$FORMAT_CLUSTER" -eq 1 ]; then + "$IMPALA_HOME/testdata/bin/run-all.sh" -format +elif [ "$TESTDATA_ACTION" -eq 1 ] || [ "$TESTS_ACTION "-eq 1 ]; then + "$IMPALA_HOME/testdata/bin/run-all.sh" fi # @@ -368,7 +368,7 @@ if [ ${IMPALA_KERBERIZE} -eq 1 ]; then echo "won't be run. The impala daemons will be started." TESTDATA_ACTION=0 TESTS_ACTION=0 - ${IMPALA_HOME}/bin/start-impala-cluster.py + "${IMPALA_HOME}/bin/start-impala-cluster.py" fi fi # END KERBEROS TODO @@ -389,34 +389,34 @@ fi if [ $TESTDATA_ACTION -eq 1 ]; then # Create testdata. - $IMPALA_HOME/bin/create_testdata.sh - cd $ROOT + "$IMPALA_HOME/bin/create_testdata.sh" + cd "$ROOT" # We have 4 cases: # - test-warehouse and metastore snapshots exists. # - Only the test-warehouse snapshot exists. # - Only the metastore snapshot exists. # - Neither of them exist. CREATE_LOAD_DATA_ARGS="" - if [[ $SNAPSHOT_FILE && $METASTORE_SNAPSHOT_FILE ]]; then + if [[ "$SNAPSHOT_FILE" && "$METASTORE_SNAPSHOT_FILE" ]]; then CREATE_LOAD_DATA_ARGS="-snapshot_file ${SNAPSHOT_FILE} -skip_metadata_load" - elif [[ $SNAPSHOT_FILE && -z $METASTORE_SNAPSHOT_FILE ]]; then + elif [[ "$SNAPSHOT_FILE" && -z "$METASTORE_SNAPSHOT_FILE" ]]; then CREATE_LOAD_DATA_ARGS="-snapshot_file ${SNAPSHOT_FILE}" - elif [[ -z $SNAPSHOT_FILE && $METASTORE_SNAPSHOT_FILE ]]; then + elif [[ -z "$SNAPSHOT_FILE" && "$METASTORE_SNAPSHOT_FILE" ]]; then CREATE_LOAD_DATA_ARGS="-skip_metadata_load -skip_snapshot_load" fi - ${IMPALA_HOME}/testdata/bin/create-load-data.sh ${CREATE_LOAD_DATA_ARGS} <<< Y + "${IMPALA_HOME}/testdata/bin/create-load-data.sh" ${CREATE_LOAD_DATA_ARGS} <<< Y fi if [ $TESTS_ACTION -eq 1 ]; then if [ $CODE_COVERAGE -eq 0 ]; then - ${IMPALA_HOME}/bin/run-all-tests.sh -e $EXPLORATION_STRATEGY + "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY else - ${IMPALA_HOME}/bin/run-all-tests.sh -e $EXPLORATION_STRATEGY -c + "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY -c fi fi # Generate list of files for Cscope to index -$IMPALA_HOME/bin/gen-cscope.sh +"$IMPALA_HOME/bin/gen-cscope.sh" if [ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]; then echo @@ -426,6 +426,6 @@ if [ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]; then echo "environment variables weren't available before the cluster" echo "was created. To pick them up, please source impala-config.sh:" echo - echo " . ${IMPALA_HOME}/bin/impala-config.sh" + echo " . \"${IMPALA_HOME}/bin/impala-config.sh\"" echo fi http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/infra/python/deps/download_requirements ---------------------------------------------------------------------- diff --git a/infra/python/deps/download_requirements b/infra/python/deps/download_requirements index e054653..26788be 100755 --- a/infra/python/deps/download_requirements +++ b/infra/python/deps/download_requirements @@ -2,15 +2,15 @@ set -euo pipefail -DIR=$(dirname "$0") +DIR="$(dirname "$0")" -pushd $DIR -PY26=$(./find_py26.py) +pushd "$DIR" +PY26="$(./find_py26.py)" # Directly download packages listed in requirements.txt, but don't install them. -$PY26 pip_download.py +"$PY26" pip_download.py # For virtualenv, other scripts rely on the .tar.gz package (not a .whl package). -$PY26 pip_download.py virtualenv 13.1.0 +"$PY26" pip_download.py virtualenv 13.1.0 # kudu-python is downloaded separately because pip install attempts to execute a # setup.py subcommand for kudu-python that can fail even if the download succeeds. -$PY26 pip_download.py kudu-python 0.1.1 +"$PY26" pip_download.py kudu-python 0.1.1 popd
