IMPALA-7399: Emit a junit xml report when trapping errors This patch will cause a junitxml file to be emitted in the case of errors in build scripts. Instead of simply echoing a message to the console, we set up a trap function that also writes out to a junit xml report that can be consumed by jenkins.impala.io.
Main things to pay attention to: - New file that gets sourced by all bash scripts when trapping within bash scripts: https://gerrit.cloudera.org/c/11257/1/bin/report_build_error.sh - Installation of the python lib into impala-python venv for use from within python files: https://gerrit.cloudera.org/c/11257/1/bin/impala-python-common.sh - Change to the generate_junitxml.py file itself, for ease of https://gerrit.cloudera.org/c/11257/1/lib/python/impala_py_lib/jenkins/generate_junitxml.py Most of the other changes are to source the new report_build_error.sh script to set up the trap function. Change-Id: Idd62045bb43357abc2b89a78afff499149d3c3fc Reviewed-on: http://gerrit.cloudera.org:8080/11257 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/6e5ec22b Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/6e5ec22b Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/6e5ec22b Branch: refs/heads/master Commit: 6e5ec22b1237f1d466c095c96a2fc1cb71ccb2d9 Parents: bb9454f Author: David Knupp <[email protected]> Authored: Thu Aug 16 17:06:04 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Aug 23 18:33:58 2018 +0000 ---------------------------------------------------------------------- bin/clean-cmake.sh | 4 +- bin/clean.sh | 4 +- bin/create-test-configuration.sh | 3 +- bin/create_testdata.sh | 3 +- bin/distcc/distcc_server_setup.sh | 3 +- bin/impala-python-common.sh | 3 +- bin/jenkins/all-tests.sh | 3 +- bin/jenkins/build-all-flag-combinations.sh | 3 +- bin/jenkins/build-only.sh | 3 +- bin/make_impala.sh | 3 +- bin/report_build_error.sh | 30 +++++++++++++++ bin/run-all-tests.sh | 3 +- bin/run-backend-tests.sh | 3 +- bin/start-catalogd.sh | 3 +- bin/start-impalad.sh | 3 +- bin/start-statestored.sh | 3 +- buildall.sh | 4 +- infra/python/bootstrap_virtualenv.py | 24 ++++++++---- .../impala_py_lib/jenkins/generate_junitxml.py | 40 ++++++++++++-------- shell/make_shell_tarball.sh | 3 +- testdata/bin/check-schema-diff.sh | 4 ++ testdata/bin/compute-table-stats.sh | 3 +- testdata/bin/copy-data-sources.sh | 3 +- testdata/bin/copy-udfs-udas.sh | 3 +- testdata/bin/create-load-data.sh | 3 +- testdata/bin/create-table-many-blocks.sh | 3 +- testdata/bin/generate-load-nested.sh | 3 +- testdata/bin/kill-all.sh | 3 +- testdata/bin/kill-hbase.sh | 3 +- testdata/bin/kill-hive-server.sh | 3 +- testdata/bin/kill-java-service.sh | 3 +- testdata/bin/kill-sentry-service.sh | 3 +- testdata/bin/load-hive-builtins.sh | 3 +- testdata/bin/load-metastore-snapshot.sh | 3 +- testdata/bin/load-test-warehouse-snapshot.sh | 3 +- testdata/bin/run-all.sh | 3 +- testdata/bin/run-hbase.sh | 3 +- testdata/bin/run-hive-server.sh | 3 +- testdata/bin/run-mini-dfs.sh | 3 +- testdata/bin/run-sentry-service.sh | 3 +- testdata/bin/setup-hdfs-env.sh | 3 +- testdata/cluster/admin | 3 +- testdata/datasets/tpcds/preload | 3 +- testdata/datasets/tpch/preload | 3 +- tests/run-custom-cluster-tests.sh | 3 +- tests/run-process-failure-tests.sh | 3 +- 46 files changed, 159 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/clean-cmake.sh ---------------------------------------------------------------------- diff --git a/bin/clean-cmake.sh b/bin/clean-cmake.sh index 92415fc..aa1ea96 100755 --- a/bin/clean-cmake.sh +++ b/bin/clean-cmake.sh @@ -20,8 +20,8 @@ # Removes artifacts generated by cmake. set -euo pipefail -trap 'echo Error in ${0} at line ${LINENO}: $(cd "'${PWD}'" && awk "NR == ${LINENO}" \ - ${0})' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error if [[ -z "${IMPALA_HOME}" || ! -d "${IMPALA_HOME}" ]]; then echo IMPALA_HOME=${IMPALA_HOME} is not valid. 1>&2 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/clean.sh ---------------------------------------------------------------------- diff --git a/bin/clean.sh b/bin/clean.sh index 89a991c..d0b7c3b 100755 --- a/bin/clean.sh +++ b/bin/clean.sh @@ -22,8 +22,8 @@ # branch to a non-toolchain branch due to caching in CMake generated files. set -euo pipefail -trap 'echo Error in ${0} at line ${LINENO}: $(cd "'${PWD}'" && awk "NR == ${LINENO}" \ - ${0})' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error # If the project was never build, no Makefile will exist and thus make clean will fail. # Combine the make command with the bash noop to always return true. http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/create-test-configuration.sh ---------------------------------------------------------------------- diff --git a/bin/create-test-configuration.sh b/bin/create-test-configuration.sh index e68af9c..6e799fb 100755 --- a/bin/create-test-configuration.sh +++ b/bin/create-test-configuration.sh @@ -22,7 +22,8 @@ # as creation of the Hive metastore. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error CREATE_METASTORE=0 CREATE_SENTRY_POLICY_DB=0 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/create_testdata.sh ---------------------------------------------------------------------- diff --git a/bin/create_testdata.sh b/bin/create_testdata.sh index cc296bf..5b81d9e 100755 --- a/bin/create_testdata.sh +++ b/bin/create_testdata.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error bin=`dirname "$0"` bin=`cd "$bin"; pwd` http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/distcc/distcc_server_setup.sh ---------------------------------------------------------------------- diff --git a/bin/distcc/distcc_server_setup.sh b/bin/distcc/distcc_server_setup.sh index 8b5e6a9..6b514e7 100755 --- a/bin/distcc/distcc_server_setup.sh +++ b/bin/distcc/distcc_server_setup.sh @@ -39,7 +39,8 @@ # CCACHE_DIR: directory to use for distccd's ccache. # CCACHE_SIZE: size of ccache, passed to ccache's -M option set -eu -o pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error if [[ $# != 1 ]]; then echo "Usage: $0 <allowed IP address range>" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/impala-python-common.sh ---------------------------------------------------------------------- diff --git a/bin/impala-python-common.sh b/bin/impala-python-common.sh index 29a36a2..501d487 100644 --- a/bin/impala-python-common.sh +++ b/bin/impala-python-common.sh @@ -19,7 +19,8 @@ # $IMPALA_HOME/bin/impala-py* executables. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error LD_LIBRARY_PATH+=":$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \ --print-ld-library-path)" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/jenkins/all-tests.sh ---------------------------------------------------------------------- diff --git a/bin/jenkins/all-tests.sh b/bin/jenkins/all-tests.sh index 7917358..1e73722 100644 --- a/bin/jenkins/all-tests.sh +++ b/bin/jenkins/all-tests.sh @@ -19,7 +19,8 @@ # Run all Impala tests. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error cd "${IMPALA_HOME}" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/jenkins/build-all-flag-combinations.sh ---------------------------------------------------------------------- diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh index 8dce06d..200729e 100755 --- a/bin/jenkins/build-all-flag-combinations.sh +++ b/bin/jenkins/build-all-flag-combinations.sh @@ -25,7 +25,8 @@ # Usage: build-all-flag-combinations.sh [--dryrun] set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error export IMPALA_MAVEN_OPTIONS="-U" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/jenkins/build-only.sh ---------------------------------------------------------------------- diff --git a/bin/jenkins/build-only.sh b/bin/jenkins/build-only.sh index d14ad6e..e7bdb3f 100644 --- a/bin/jenkins/build-only.sh +++ b/bin/jenkins/build-only.sh @@ -19,7 +19,8 @@ # Only run an Impala build. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error cd "${IMPALA_HOME}" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/make_impala.sh ---------------------------------------------------------------------- diff --git a/bin/make_impala.sh b/bin/make_impala.sh index 8e1807b..f33bd74 100755 --- a/bin/make_impala.sh +++ b/bin/make_impala.sh @@ -20,7 +20,8 @@ # Incrementally compiles the frontend and backend. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error : ${IMPALA_TOOLCHAIN=} http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/report_build_error.sh ---------------------------------------------------------------------- diff --git a/bin/report_build_error.sh b/bin/report_build_error.sh new file mode 100644 index 0000000..295cce0 --- /dev/null +++ b/bin/report_build_error.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +report_build_error() { + ERROR_MSG=$(cd "$PWD" && awk "NR == $1" $0) + FILENAME=$(basename -- "$0") + echo ERROR in $0 at line $1: $ERROR_MSG + $IMPALA_HOME/bin/generate_junitxml.py --step "${FILENAME%.*}" \ + --error "Error in $0 at line $1: $ERROR_MSG" +} + +setup_report_build_error() { + trap 'report_build_error $LINENO' ERR +} http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/run-all-tests.sh ---------------------------------------------------------------------- diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh index 5f6831e..da7090e 100755 --- a/bin/run-all-tests.sh +++ b/bin/run-all-tests.sh @@ -22,7 +22,8 @@ # Exit on reference to uninitialized variables and non-zero exit codes set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . "$IMPALA_HOME/bin/set-pythonpath.sh" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/run-backend-tests.sh ---------------------------------------------------------------------- diff --git a/bin/run-backend-tests.sh b/bin/run-backend-tests.sh index 3bc84c2..16a432f 100755 --- a/bin/run-backend-tests.sh +++ b/bin/run-backend-tests.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error export GTEST_OUTPUT="xml:$IMPALA_BE_TEST_LOGS_DIR/" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/start-catalogd.sh ---------------------------------------------------------------------- diff --git a/bin/start-catalogd.sh b/bin/start-catalogd.sh index a8b7e28..35cf592 100755 --- a/bin/start-catalogd.sh +++ b/bin/start-catalogd.sh @@ -21,7 +21,8 @@ # -build_type parameter can be passed to determine the build type to use. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error BUILD_TYPE=latest CATALOGD_ARGS="" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/start-impalad.sh ---------------------------------------------------------------------- diff --git a/bin/start-impalad.sh b/bin/start-impalad.sh index f052fa2..dacd44c 100755 --- a/bin/start-impalad.sh +++ b/bin/start-impalad.sh @@ -21,7 +21,8 @@ # parameter can be passed to determine the build type to use for the impalad instance. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error BUILD_TYPE=latest IMPALAD_ARGS="" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/start-statestored.sh ---------------------------------------------------------------------- diff --git a/bin/start-statestored.sh b/bin/start-statestored.sh index f023810..eb94910 100755 --- a/bin/start-statestored.sh +++ b/bin/start-statestored.sh @@ -20,7 +20,8 @@ # Starts up the StateStored with the specified command line arguments. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error BUILD_TYPE=latest STATESTORED_ARGS="" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/buildall.sh ---------------------------------------------------------------------- diff --git a/buildall.sh b/buildall.sh index 59e27ee..339168e 100755 --- a/buildall.sh +++ b/buildall.sh @@ -18,6 +18,8 @@ # under the License. set -euo pipefail +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error # run buildall.sh -help to see options ROOT=`dirname "$0"` @@ -29,8 +31,6 @@ then exit 1 fi -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR - # Grab this *before* we source impala-config.sh to see if the caller has # kerberized environment variables already or not. NEEDS_RE_SOURCE_NOTE=1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/infra/python/bootstrap_virtualenv.py ---------------------------------------------------------------------- diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py index 1953935..f2ce839 100644 --- a/infra/python/bootstrap_virtualenv.py +++ b/infra/python/bootstrap_virtualenv.py @@ -140,29 +140,37 @@ def exec_pip_install(args, cc="no-cc-available", env=None): # Don't call the virtualenv pip directly, it uses a hashbang to to call the python # virtualenv using an absolute path. If the path to the virtualenv is very long, the # hashbang won't work. - # + impala_pip_base_cmd = [os.path.join(ENV_DIR, "bin", "python"), + os.path.join(ENV_DIR, "bin", "pip"), "install", "-v"] + # Passes --no-binary for IMPALA-3767: without this, Cython (and # several other packages) fail download. # # --no-cache-dir is used to prevent caching of compiled artifacts, which may be built # with different compilers or settings. - cmd = [os.path.join(ENV_DIR, "bin", "python"), os.path.join(ENV_DIR, "bin", "pip"), - "install", "-v", "--no-binary", ":all:", "--no-cache-dir"] + third_party_pkg_install_cmd = \ + impala_pip_base_cmd[:] + ["--no-binary", ":all:", "--no-cache-dir"] # When using a custom mirror, we also must use the index of that mirror. if "PYPI_MIRROR" in os.environ: - cmd.extend(["--index-url", "%s/simple" % os.environ["PYPI_MIRROR"]]) + third_party_pkg_install_cmd.extend(["--index-url", + "%s/simple" % os.environ["PYPI_MIRROR"]]) else: # Prevent fetching additional packages from the index. If we forget to add a package # to one of the requirements.txt files, this should trigger an error. However, we will # still access the index for version/dependency resolution, hence we need to change it # when using a private mirror. - cmd.append("--no-index") + third_party_pkg_install_cmd.append("--no-index") - cmd.extend(["--find-links", + third_party_pkg_install_cmd.extend(["--find-links", "file://%s" % urllib.pathname2url(os.path.abspath(DEPS_DIR))]) - cmd.extend(args) - exec_cmd(cmd, env=env) + third_party_pkg_install_cmd.extend(args) + exec_cmd(third_party_pkg_install_cmd, env=env) + + # Finally, we want to install the packages from our own internal python lib + local_package_install_cmd = impala_pip_base_cmd + \ + ['-e', os.path.join(os.getenv('IMPALA_HOME'), 'lib', 'python')] + exec_cmd(local_package_install_cmd) def find_file(*paths): http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/lib/python/impala_py_lib/jenkins/generate_junitxml.py ---------------------------------------------------------------------- diff --git a/lib/python/impala_py_lib/jenkins/generate_junitxml.py b/lib/python/impala_py_lib/jenkins/generate_junitxml.py index 68bedd1..97dcbdd 100755 --- a/lib/python/impala_py_lib/jenkins/generate_junitxml.py +++ b/lib/python/impala_py_lib/jenkins/generate_junitxml.py @@ -32,18 +32,30 @@ from datetime import datetime as dt IMPALA_HOME = os.getenv('IMPALA_HOME', '.') SCRIPT_NAME, _ = os.path.splitext(os.path.basename(__file__)) +JUNITXML_LOGDIR = os.path.join(os.getenv("IMPALA_LOGS_DIR", "."), 'extra_junit_xml_logs') class JunitReport(object): """A Junit XML style report parseable by Jenkins for reporting build status. - Generally, a caller who invokes this script doesn't need to do anything + Generally, a caller who invokes this script from bash doesn't need to do more than supply the necessary command line parameters. The JunitReport class is instantiated using those initial inputs, and a timestamped XML file is output to the $IMPALA_HOME/logs/extra_junit_xml_logs/. Log files are timestamped, so they will not overwrite previous files containing output of the same step. + + For use from within a python script (must be invoked with impala-python), an + example might look like: + + >>> from impala_py_lib.jenkins.generate_junitxml import JunitReport + >>> report = JunitReport(phase='load_data', step='load_hbase', error_msg='oops') + >>> report.tofile() + + For now, the class does not support adding more than one step (analogous to a + test case) to the same phase (analogous to a test suite). Each report should + be unique for a given junit XML file. This may be enhanced at some point. """ def __init__(self, phase, step, error_msg=None, stdout=None, stderr=None, @@ -124,7 +136,7 @@ class JunitReport(object): output = ET.SubElement(self.testcase_element, "system-{}".format(output_type)) output.text = JunitReport.get_xml_content(file_or_string) - def to_file(self, junitxml_logdir='.'): + def to_file(self, junitxml_logdir=JUNITXML_LOGDIR): """ Create a timestamped XML report file. @@ -134,6 +146,15 @@ class JunitReport(object): Return: junit_log_file: path to the generated file """ + # The equivalent of mkdir -p + try: + os.makedirs(junitxml_logdir) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(junitxml_logdir): + pass + else: + raise + filename = '{}.{}.xml'.format( self.testsuite_element.attrib['name'], self.utc_time.strftime('%Y%m%d_%H_%M_%S') @@ -226,17 +247,6 @@ def main(): Phase can be repeated in a given test run, but the step leaf node, which is equivalent to a "test case", must be unique within each phase. """ - junitxml_logdir = os.path.join(IMPALA_HOME, 'logs', 'extra_junit_xml_logs') - - # The equivalent of mkdir -p - try: - os.makedirs(junitxml_logdir) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(junitxml_logdir): - pass - else: - raise - options = get_options() junit_report = JunitReport(phase=options.phase, @@ -246,8 +256,8 @@ def main(): stderr=options.stderr, elapsed_time=options.time) - xml_report = junit_report.to_file(junitxml_logdir) - print("Generated: {}".format(xml_report)) + junit_log_file = junit_report.to_file() + print("Generated: {0}".format(junit_log_file)) if "__main__" == __name__: http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/shell/make_shell_tarball.sh ---------------------------------------------------------------------- diff --git a/shell/make_shell_tarball.sh b/shell/make_shell_tarball.sh index 637fa7d..dd6a648 100755 --- a/shell/make_shell_tarball.sh +++ b/shell/make_shell_tarball.sh @@ -23,7 +23,8 @@ # ${IMPALA_HOME}/shell/build. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error if [ "x${IMPALA_HOME}" == "x" ]; then echo "\$IMPALA_HOME must be set" http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/check-schema-diff.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/check-schema-diff.sh b/testdata/bin/check-schema-diff.sh index 4eb8172..dd36b85 100755 --- a/testdata/bin/check-schema-diff.sh +++ b/testdata/bin/check-schema-diff.sh @@ -24,6 +24,10 @@ # - 1 implies that the schemas have changed. set -euo pipefail + +# We don't want to generate a junit xml report for errors generated here, +# since exit code 1 here denotes something useful. So in the case of this +# script, we don't call setup_report_build_error. trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/compute-table-stats.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/compute-table-stats.sh b/testdata/bin/compute-table-stats.sh index 63eb0da..08c7595 100755 --- a/testdata/bin/compute-table-stats.sh +++ b/testdata/bin/compute-table-stats.sh @@ -20,7 +20,8 @@ # Runs compute table stats over a curated set of Impala test tables. # set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/copy-data-sources.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/copy-data-sources.sh b/testdata/bin/copy-data-sources.sh index a1838ce..1782aca 100755 --- a/testdata/bin/copy-data-sources.sh +++ b/testdata/bin/copy-data-sources.sh @@ -20,7 +20,8 @@ # This script copies the test data source library into hdfs. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/copy-udfs-udas.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh index c1a0454..def7812 100755 --- a/testdata/bin/copy-udfs-udas.sh +++ b/testdata/bin/copy-udfs-udas.sh @@ -20,7 +20,8 @@ # This script copies udf/uda binaries into hdfs. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error if [ x${JAVA_HOME} == x ]; then echo JAVA_HOME not set http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/create-load-data.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh index 1953daf..74ae248 100755 --- a/testdata/bin/create-load-data.sh +++ b/testdata/bin/create-load-data.sh @@ -29,7 +29,8 @@ # bin/load-data.py set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 . ${IMPALA_HOME}/testdata/bin/run-step.sh http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/create-table-many-blocks.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/create-table-many-blocks.sh b/testdata/bin/create-table-many-blocks.sh index 4c0a57d..2db9bee 100755 --- a/testdata/bin/create-table-many-blocks.sh +++ b/testdata/bin/create-table-many-blocks.sh @@ -25,7 +25,8 @@ # blocks/files. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/generate-load-nested.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/generate-load-nested.sh b/testdata/bin/generate-load-nested.sh index 4986418..cceea1a 100755 --- a/testdata/bin/generate-load-nested.sh +++ b/testdata/bin/generate-load-nested.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error SHELL_CMD=${IMPALA_HOME}/bin/impala-shell.sh http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-all.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/kill-all.sh b/testdata/bin/kill-all.sh index 0e8d201..b6c13a6 100755 --- a/testdata/bin/kill-all.sh +++ b/testdata/bin/kill-all.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error # Shutdown Impala if it is alive ${IMPALA_HOME}/bin/start-impala-cluster.py --kill http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-hbase.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/kill-hbase.sh b/testdata/bin/kill-hbase.sh index 4e5a42f..0a5fcda 100755 --- a/testdata/bin/kill-hbase.sh +++ b/testdata/bin/kill-hbase.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error DIR=$(dirname "$0") echo Stopping Hbase http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-hive-server.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/kill-hive-server.sh b/testdata/bin/kill-hive-server.sh index 0ebd18c..59c44e5 100755 --- a/testdata/bin/kill-hive-server.sh +++ b/testdata/bin/kill-hive-server.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error DIR=$(dirname "$0") echo Stopping Hive http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-java-service.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/kill-java-service.sh b/testdata/bin/kill-java-service.sh index 4cdc1bd..8ae449d 100755 --- a/testdata/bin/kill-java-service.sh +++ b/testdata/bin/kill-java-service.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error CLASSES=() EXTRA_SHUTDOWN_TIME_SECS=1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-sentry-service.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/kill-sentry-service.sh b/testdata/bin/kill-sentry-service.sh index e9ed7aa..ceb8958 100755 --- a/testdata/bin/kill-sentry-service.sh +++ b/testdata/bin/kill-sentry-service.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error DIR=$(dirname "$0") echo Stopping Sentry http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/load-hive-builtins.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/load-hive-builtins.sh b/testdata/bin/load-hive-builtins.sh index 061d42b..55cc845 100755 --- a/testdata/bin/load-hive-builtins.sh +++ b/testdata/bin/load-hive-builtins.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/load-metastore-snapshot.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/load-metastore-snapshot.sh b/testdata/bin/load-metastore-snapshot.sh index 1760cdd..dd4e136 100755 --- a/testdata/bin/load-metastore-snapshot.sh +++ b/testdata/bin/load-metastore-snapshot.sh @@ -22,7 +22,8 @@ # full data load build. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/load-test-warehouse-snapshot.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/load-test-warehouse-snapshot.sh b/testdata/bin/load-test-warehouse-snapshot.sh index fe5dd2a..da5fb07 100755 --- a/testdata/bin/load-test-warehouse-snapshot.sh +++ b/testdata/bin/load-test-warehouse-snapshot.sh @@ -25,7 +25,8 @@ # to backup any data you need before running this script. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 : ${REMOTE_LOAD:=} http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-all.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/run-all.sh b/testdata/bin/run-all.sh index 6820e5d..1e14315 100755 --- a/testdata/bin/run-all.sh +++ b/testdata/bin/run-all.sh @@ -20,7 +20,8 @@ # Starts up a mini-dfs test cluster and related services set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error # If -format is passed, format the mini-dfs cluster. http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-hbase.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/run-hbase.sh b/testdata/bin/run-hbase.sh index 1433073..e7d67c9 100755 --- a/testdata/bin/run-hbase.sh +++ b/testdata/bin/run-hbase.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error CLUSTER_BIN=${IMPALA_HOME}/testdata/bin HBASE_JAAS_CLIENT=${HBASE_CONF_DIR}/hbase-jaas-client.conf http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-hive-server.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/run-hive-server.sh b/testdata/bin/run-hive-server.sh index 2b5a486..fbeba72 100755 --- a/testdata/bin/run-hive-server.sh +++ b/testdata/bin/run-hive-server.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/set-pythonpath.sh http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-mini-dfs.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/run-mini-dfs.sh b/testdata/bin/run-mini-dfs.sh index ea6c519..be63715 100755 --- a/testdata/bin/run-mini-dfs.sh +++ b/testdata/bin/run-mini-dfs.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error if [[ $# -eq 1 && "$1" == -format ]]; then SHOULD_FORMAT=true http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-sentry-service.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/run-sentry-service.sh b/testdata/bin/run-sentry-service.sh index 755c382..f49f88b 100755 --- a/testdata/bin/run-sentry-service.sh +++ b/testdata/bin/run-sentry-service.sh @@ -18,7 +18,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error . ${IMPALA_HOME}/bin/set-classpath.sh http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/setup-hdfs-env.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/setup-hdfs-env.sh b/testdata/bin/setup-hdfs-env.sh index 552c48b..4308757 100755 --- a/testdata/bin/setup-hdfs-env.sh +++ b/testdata/bin/setup-hdfs-env.sh @@ -18,7 +18,8 @@ # under the License. # set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error : ${REMOTE_LOAD:=} http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/cluster/admin ---------------------------------------------------------------------- diff --git a/testdata/cluster/admin b/testdata/cluster/admin index 534bc45..ca438db 100755 --- a/testdata/cluster/admin +++ b/testdata/cluster/admin @@ -28,7 +28,8 @@ # TODO: Run each node on its own IP address, e.g. 127.0.0.1, 127.0.0.2, and so on. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error : ${IMPALA_KERBERIZE=} : ${INCLUDE_YARN=} http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/datasets/tpcds/preload ---------------------------------------------------------------------- diff --git a/testdata/datasets/tpcds/preload b/testdata/datasets/tpcds/preload index 631a1c2..423b3c8 100755 --- a/testdata/datasets/tpcds/preload +++ b/testdata/datasets/tpcds/preload @@ -17,7 +17,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error IMPALA_DATA=${IMPALA_HOME}/testdata/impala-data TPC_DS_DATA=${IMPALA_DATA}/tpcds http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/datasets/tpch/preload ---------------------------------------------------------------------- diff --git a/testdata/datasets/tpch/preload b/testdata/datasets/tpch/preload index 2b0cbb6..619a4d2 100755 --- a/testdata/datasets/tpch/preload +++ b/testdata/datasets/tpch/preload @@ -17,7 +17,8 @@ # under the License. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error IMPALA_DATA=${IMPALA_HOME}/testdata/impala-data TPC_H_DATA=${IMPALA_DATA}/tpch http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/tests/run-custom-cluster-tests.sh ---------------------------------------------------------------------- diff --git a/tests/run-custom-cluster-tests.sh b/tests/run-custom-cluster-tests.sh index ce161b1..6b77e26 100755 --- a/tests/run-custom-cluster-tests.sh +++ b/tests/run-custom-cluster-tests.sh @@ -21,7 +21,8 @@ # clusters will be restarted. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error # Disable HEAPCHECK for the process failure tests because they can cause false positives. # TODO: Combine with run-process-failure-tests.sh http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/tests/run-process-failure-tests.sh ---------------------------------------------------------------------- diff --git a/tests/run-process-failure-tests.sh b/tests/run-process-failure-tests.sh index 9ab531c..db47571 100755 --- a/tests/run-process-failure-tests.sh +++ b/tests/run-process-failure-tests.sh @@ -20,7 +20,8 @@ # Runs the Impala process failure tests. set -euo pipefail -trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error # Disable HEAPCHECK for the process failure tests because they can cause false positives. export HEAPCHECK=
