IMPALA-3892: qgen: always run Impala with -convert_legacy_hive_parquet_utc_timestamps=true
To reduce discrepancies in timestamp columns between Impala and PostgresQL, always run Impala with -convert_legacy_hive_parquet_utc_timestamps=true . This makes triaging correctness issues easier. Change-Id: I6e0a14780dd9bcc18285f2857625fd3ec1c34913 Reviewed-on: http://gerrit.cloudera.org:8080/3713 Reviewed-by: Michael Brown <[email protected]> Tested-by: Michael Brown <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/af8b1876 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/af8b1876 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/af8b1876 Branch: refs/heads/master Commit: af8b187660cddba3ec7b4af772dfa2c760076c84 Parents: 72e3c14 Author: Michael Brown <[email protected]> Authored: Fri Apr 15 11:20:47 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Fri Jul 22 11:03:33 2016 -0700 ---------------------------------------------------------------------- tests/comparison/discrepancy_searcher.py | 14 ++++++++++---- tests/comparison/leopard/impala_docker_env.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af8b1876/tests/comparison/discrepancy_searcher.py ---------------------------------------------------------------------- diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py index 0ff0fee..bed551f 100755 --- a/tests/comparison/discrepancy_searcher.py +++ b/tests/comparison/discrepancy_searcher.py @@ -641,15 +641,21 @@ class QueryResultDiffSearcher(object): # Assume Impala crashed and try restarting test_crash_count += 1 LOG.info('Restarting Impala') - call([join_path(getenv('IMPALA_HOME'), 'bin/start-impala-cluster.py'), - '--log_dir=%s' % getenv('LOG_DIR', "/tmp/")]) + impalad_args = [ + '-convert_legacy_hive_parquet_utc_timestamps=true', + ] + impala_restart_cmd = [ + join_path(getenv('IMPALA_HOME'), 'bin/start-impala-cluster.py'), + '--log_dir={0}'.format(getenv('LOG_DIR', "/tmp/")), + '--impalad_args="{0}"'.format(' '.join(impalad_args)), + ] + call(impala_restart_cmd) self.test_conn.reconnect() query_result_comparator.test_cursor = self.test_conn.cursor() result = query_result_comparator.compare_query_results(query) if result.error: LOG.info('Restarting Impala') - call([join_path(getenv('IMPALA_HOME'), 'bin/start-impala-cluster.py'), - '--log_dir=%s' % getenv('LOG_DIR', "/tmp/")]) + call(impala_restart_cmd) self.test_conn.reconnect() query_result_comparator.test_cursor = self.test_conn.cursor() else: http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af8b1876/tests/comparison/leopard/impala_docker_env.py ---------------------------------------------------------------------- diff --git a/tests/comparison/leopard/impala_docker_env.py b/tests/comparison/leopard/impala_docker_env.py index 1a374f1..9b17420 100755 --- a/tests/comparison/leopard/impala_docker_env.py +++ b/tests/comparison/leopard/impala_docker_env.py @@ -184,8 +184,14 @@ class ImpalaDockerEnv(object): warn_only = True, host_string = '{0}@{1}:{2}'.format(DOCKER_USER_NAME, self.host, self.ssh_port), password = os.environ['DOCKER_PASSWORD']): - start_command = ('source {IMPALA_HOME}/bin/impala-config.sh ' - '&& {IMPALA_HOME}/bin/start-impala-cluster.py').format(IMPALA_HOME = IMPALA_HOME) + impalad_args = [ + '-convert_legacy_hive_parquet_utc_timestamps=true', + ] + start_command = ( + 'source {IMPALA_HOME}/bin/impala-config.sh ' + '&& {IMPALA_HOME}/bin/start-impala-cluster.py ' + '--impalad_args="{impalad_args}"').format(IMPALA_HOME=IMPALA_HOME, + impalad_args=' '.join(impalad_args)) result = retry(run)(start_command, pty=False) return result
