IMPALA-7017: deflake/fix test_catalog_restart test The custom_cluster/test_metadata_replicas.py:test_catalog_restart test has been recently flaky/broken for two reasons:
1) Variable support for Hive and non-hdfs filesystems. Other tests that depend on Hive have disabled tests for non-hdfs filesystems. Since the functionality tested is not intended for all filesystems, this change disables this test for all filesystems other than hdfs. 2) Several builds have been flaky when looking up catalogd's version. This change adds a retry for obtaining the version. Change-Id: Iab6edb01f0bd7f5408cfef28fd05fdc95fb78469 Reviewed-on: http://gerrit.cloudera.org:8080/10397 Reviewed-by: Joe McDonnell <joemcdonn...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/6af65697 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/6af65697 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/6af65697 Branch: refs/heads/master Commit: 6af65697f291b859509d756b1d839176f664111d Parents: 1e6544f Author: Vuk Ercegovac <vercego...@cloudera.com> Authored: Mon May 14 15:06:33 2018 -0700 Committer: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Committed: Thu May 17 09:01:14 2018 +0000 ---------------------------------------------------------------------- tests/common/impala_service.py | 16 +++++++++++++--- tests/custom_cluster/test_metadata_replicas.py | 9 +++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/6af65697/tests/common/impala_service.py ---------------------------------------------------------------------- diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py index bc5b4a3..9772b13 100644 --- a/tests/common/impala_service.py +++ b/tests/common/impala_service.py @@ -308,6 +308,16 @@ class CatalogdService(BaseImpalaService): super(CatalogdService, self).__init__(hostname, webserver_port) self.service_port = service_port - def get_catalog_version(self): - """ Gets catalogd's latest catalog version. """ - return self.get_debug_webpage_json('catalog')["version"] + def get_catalog_version(self, timeout=10, interval=1): + """ Gets catalogd's latest catalog version. Retry for 'timeout' + seconds, sleeping 'interval' seconds between tries. If the + version cannot be obtained, this method fails.""" + start_time = time() + while (time() - start_time < timeout): + try: + info = self.get_debug_webpage_json('catalog') + if "version" in info: return info['version'] + except Exception: + LOG.info('Catalogd version not yet available.') + sleep(interval) + assert False, 'Catalog version not ready in expected time.' http://git-wip-us.apache.org/repos/asf/impala/blob/6af65697/tests/custom_cluster/test_metadata_replicas.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_metadata_replicas.py b/tests/custom_cluster/test_metadata_replicas.py index 589bece..9674756 100644 --- a/tests/custom_cluster/test_metadata_replicas.py +++ b/tests/custom_cluster/test_metadata_replicas.py @@ -20,8 +20,17 @@ import re from time import sleep from tests.common.environ import specific_build_type_timeout from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.skip import ( + SkipIfS3, + SkipIfADLS, + SkipIfIsilon, + SkipIfLocal) from tests.util.hive_utils import HiveDbWrapper +@SkipIfS3.hive +@SkipIfADLS.hive +@SkipIfIsilon.hive +@SkipIfLocal.hive class TestMetadataReplicas(CustomClusterTestSuite): """ Validates metadata content across catalogd and impalad coordinators."""