IMPALA-7185: low statestore custom cluster interval This changes the default statestore interval for the custom cluster tests. This can reduce the time taken for the cluster to start and metadata to load. On some tests this resulted in saving 5+ seconds per test. Overall it shaved around a minute off the custom cluster tests.
Testing: Ran 10 iterations of the tests. Change-Id: Ia5d1612283ff420d95b0dd0ca5a2a67f56765f79 Reviewed-on: http://gerrit.cloudera.org:8080/10845 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e07fbc1b Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e07fbc1b Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e07fbc1b Branch: refs/heads/master Commit: e07fbc1b6342836b3e2e472a91c61c5f75d48733 Parents: 010321d Author: Tim Armstrong <[email protected]> Authored: Mon Jun 18 14:28:13 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Mon Jul 2 22:06:38 2018 +0000 ---------------------------------------------------------------------- tests/authorization/test_grant_revoke.py | 7 ++----- tests/common/custom_cluster_test_suite.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/e07fbc1b/tests/authorization/test_grant_revoke.py ---------------------------------------------------------------------- diff --git a/tests/authorization/test_grant_revoke.py b/tests/authorization/test_grant_revoke.py index 34ee513..36642fb 100644 --- a/tests/authorization/test_grant_revoke.py +++ b/tests/authorization/test_grant_revoke.py @@ -100,8 +100,7 @@ class TestGrantRevoke(CustomClusterTestSuite, ImpalaTestSuite): @CustomClusterTestSuite.with_args( impalad_args="--server_name=server1", catalogd_args="--sentry_config=" + SENTRY_CONFIG_FILE + - " --sentry_catalog_polling_frequency_s=1", - statestored_args="--statestore_update_frequency_ms=300") + " --sentry_catalog_polling_frequency_s=1") def test_role_privilege_case(self, vector): """IMPALA-5582: Store sentry privileges in lower case. This test grants select privileges to roles assgined to tables/db @@ -165,9 +164,7 @@ class TestGrantRevoke(CustomClusterTestSuite, ImpalaTestSuite): @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impalad_args="--server_name=server1", - catalogd_args="--sentry_config=" + SENTRY_CONFIG_FILE, - statestored_args=("--statestore_heartbeat_frequency_ms=300 " - "--statestore_update_frequency_ms=300")) + catalogd_args="--sentry_config=" + SENTRY_CONFIG_FILE) def test_role_update(self, vector): """IMPALA-5355: The initial update from the statestore has the privileges and roles in reverse order if a role was modified, but not the associated privilege. Verify that http://git-wip-us.apache.org/repos/asf/impala/blob/e07fbc1b/tests/common/custom_cluster_test_suite.py ---------------------------------------------------------------------- diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py index 51e3f8f..a5c285d 100644 --- a/tests/common/custom_cluster_test_suite.py +++ b/tests/common/custom_cluster_test_suite.py @@ -39,6 +39,11 @@ CATALOGD_ARGS = 'catalogd_args' # Additional args passed to the start-impala-cluster script. START_ARGS = 'start_args' +# Run with fast topic updates by default to reduce time to first query running. +DEFAULT_STATESTORE_ARGS = '--statestore_update_frequency_ms=50 \ + --statestore_priority_update_frequency_ms=50 \ + --statestore_heartbeat_frequency_ms=50' + class CustomClusterTestSuite(ImpalaTestSuite): """Every test in a test suite deriving from this class gets its own Impala cluster. Custom arguments may be passed to the cluster by using the @with_args decorator.""" @@ -88,8 +93,11 @@ class CustomClusterTestSuite(ImpalaTestSuite): def decorate(func): if impalad_args is not None: func.func_dict[IMPALAD_ARGS] = impalad_args - if statestored_args is not None: - func.func_dict[STATESTORED_ARGS] = statestored_args + if statestored_args is None: + func.func_dict[STATESTORED_ARGS] = DEFAULT_STATESTORE_ARGS + else: + func.func_dict[STATESTORED_ARGS] = \ + DEFAULT_STATESTORE_ARGS + " " + statestored_args if catalogd_args is not None: func.func_dict[CATALOGD_ARGS] = catalogd_args if start_args is not None:
