This is an automated email from the ASF dual-hosted git repository. jbapple pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit fa672909c868f76aa50e9fb756114c32daaf6d9b Author: Jim Apple <jbapple-imp...@apache.org> AuthorDate: Fri Jan 25 16:49:19 2019 -0800 IMPALA-8062: Call impala-config in single_node_perf_run This wraps most shell calls in single_node_perf_run.py with a bash shell that first sources impala-config.sh, to make sure environment variables are set properly. Change-Id: Ic7c1b77906a975c37f3b51a0f900ed3536b398ba Reviewed-on: http://gerrit.cloudera.org:8080/12277 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> --- bin/single_node_perf_run.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/bin/single_node_perf_run.py b/bin/single_node_perf_run.py index c931266..4966614 100755 --- a/bin/single_node_perf_run.py +++ b/bin/single_node_perf_run.py @@ -67,6 +67,7 @@ from tempfile import mkdtemp import json import os +import pipes import sh import shutil import subprocess @@ -78,16 +79,24 @@ from tests.common.test_dimensions import TableFormatInfo IMPALA_HOME = os.environ["IMPALA_HOME"] +def configured_call(cmd): + """Call a command in a shell with config-impala.sh.""" + if type(cmd) is list: + cmd = " ".join([pipes.quote(arg) for arg in cmd]) + cmd = "source {0}/bin/impala-config.sh && {1}".format(IMPALA_HOME, cmd) + return subprocess.check_call(["bash", "-c", cmd]) + + def load_data(db_to_load, table_formats, scale): """Loads a database with a particular scale factor.""" - subprocess.check_call(["{0}/bin/load-data.py".format(IMPALA_HOME), - "--workloads", db_to_load, "--scale_factor", str(scale), - "--table_formats", "text/none," + table_formats]) + configured_call(["{0}/bin/load-data.py".format(IMPALA_HOME), + "--workloads", db_to_load, "--scale_factor", str(scale), + "--table_formats", "text/none," + table_formats]) for table_format in table_formats.split(","): suffix = TableFormatInfo.create_from_string(None, table_format).db_suffix() db_name = db_to_load + scale + suffix - subprocess.check_call(["{0}/tests/util/compute_table_stats.py".format(IMPALA_HOME), - "--stop_on_error", "--db_names", db_name]) + configured_call(["{0}/tests/util/compute_table_stats.py".format(IMPALA_HOME), + "--stop_on_error", "--db_names", db_name]) def get_git_hash_for_name(name): return sh.git("rev-parse", name).strip() @@ -99,17 +108,17 @@ def build(git_hash, options): buildall = ["{0}/buildall.sh".format(IMPALA_HOME), "-notests", "-release", "-noclean"] if options.ninja: buildall += ["-ninja"] - subprocess.check_call(buildall) + configured_call(buildall) def start_minicluster(): - subprocess.check_call(["{0}/bin/create-test-configuration.sh".format(IMPALA_HOME)]) - subprocess.check_call(["{0}/testdata/bin/run-all.sh".format(IMPALA_HOME)]) + configured_call(["{0}/bin/create-test-configuration.sh".format(IMPALA_HOME)]) + configured_call(["{0}/testdata/bin/run-all.sh".format(IMPALA_HOME)]) def start_impala(num_impalads): - subprocess.check_call(["{0}/bin/start-impala-cluster.py".format(IMPALA_HOME), "-s", - str(num_impalads), "-c", str(num_impalads)]) + configured_call(["{0}/bin/start-impala-cluster.py".format(IMPALA_HOME), "-s", + str(num_impalads), "-c", str(num_impalads)]) def run_workload(base_dir, workloads, options): @@ -134,7 +143,7 @@ def run_workload(base_dir, workloads, options): if options.query_names: run_workload += ["--query_names={0}".format(options.query_names)] - subprocess.check_call(run_workload) + configured_call(run_workload) def report_benchmark_results(file_a, file_b, description):