Repository: impala Updated Branches: refs/heads/master 004719126 -> af76186e0
IMPALA-7554: Update custom cluster tests to have new logs for sentry This patch adds the ability to create a new log for each spawn of the sentry service. This will enable better trouble shooting for the custom cluster tests that restart the sentry service. Testing: - Ran all custom cluster tests. Change-Id: I6e538af7fd6e6ea21dc3f4442bdebf3b31558516 Reviewed-on: http://gerrit.cloudera.org:8080/11624 Reviewed-by: Fredy Wijaya <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/21f521a7 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/21f521a7 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/21f521a7 Branch: refs/heads/master Commit: 21f521a7c280031e33cde7c61a979683c5abed00 Parents: 0047191 Author: Adam Holley <[email protected]> Authored: Thu Oct 4 22:55:54 2018 -0500 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Oct 12 01:00:56 2018 +0000 ---------------------------------------------------------------------- testdata/bin/run-sentry-service.sh | 8 +- tests/authorization/test_authorization.py | 6 +- tests/authorization/test_owner_privileges.py | 99 +++++++++++++---------- tests/common/custom_cluster_test_suite.py | 29 ++++--- tests/custom_cluster/test_redaction.py | 2 +- 5 files changed, 87 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/21f521a7/testdata/bin/run-sentry-service.sh ---------------------------------------------------------------------- diff --git a/testdata/bin/run-sentry-service.sh b/testdata/bin/run-sentry-service.sh index a508ffd..7c6167c 100755 --- a/testdata/bin/run-sentry-service.sh +++ b/testdata/bin/run-sentry-service.sh @@ -24,13 +24,19 @@ setup_report_build_error . ${IMPALA_HOME}/bin/set-classpath.sh SENTRY_SERVICE_CONFIG=${SENTRY_SERVICE_CONFIG:-} +SENTRY_LOG_DIR=${SENTRY_LOG_DIR:-} if [ -z ${SENTRY_SERVICE_CONFIG} ] then SENTRY_SERVICE_CONFIG=${SENTRY_CONF_DIR}/sentry-site.xml fi -LOGDIR="${IMPALA_CLUSTER_LOGS_DIR}"/sentry +if [ -z ${SENTRY_LOG_DIR} ] +then + LOGDIR="${IMPALA_CLUSTER_LOGS_DIR}"/sentry +else + LOGDIR=${SENTRY_LOG_DIR} +fi mkdir -p "${LOGDIR}" || true http://git-wip-us.apache.org/repos/asf/impala/blob/21f521a7/tests/authorization/test_authorization.py ---------------------------------------------------------------------- diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py index 75c0997..a508440 100644 --- a/tests/authorization/test_authorization.py +++ b/tests/authorization/test_authorization.py @@ -367,7 +367,8 @@ class TestAuthorization(CustomClusterTestSuite): @CustomClusterTestSuite.with_args( impalad_args="--server_name=server1 --sentry_config=" + SENTRY_CONFIG_FILE, catalogd_args="--sentry_config=" + SENTRY_CONFIG_FILE, - log_dir=tempfile.mkdtemp(prefix="test_deprecated_none_", dir=os.getenv("LOG_DIR"))) + impala_log_dir=tempfile.mkdtemp(prefix="test_deprecated_none_", + dir=os.getenv("LOG_DIR"))) def test_deprecated_flag_doesnt_show(self): assert_no_files_in_dir_contain(self.impala_log_dir, "authorization_policy_file " + "flag is deprecated. Object Ownership feature is not supported") @@ -377,7 +378,8 @@ class TestAuthorization(CustomClusterTestSuite): --authorization_policy_file=%s\ --authorization_policy_provider_class=%s" % (AUTH_POLICY_FILE, "org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider"), - log_dir=tempfile.mkdtemp(prefix="test_deprecated_", dir=os.getenv("LOG_DIR"))) + impala_log_dir=tempfile.mkdtemp(prefix="test_deprecated_", + dir=os.getenv("LOG_DIR"))) def test_deprecated_flags(self): assert_file_in_dir_contains(self.impala_log_dir, "authorization_policy_file flag" + " is deprecated. Object Ownership feature is not supported") http://git-wip-us.apache.org/repos/asf/impala/blob/21f521a7/tests/authorization/test_owner_privileges.py ---------------------------------------------------------------------- diff --git a/tests/authorization/test_owner_privileges.py b/tests/authorization/test_owner_privileges.py index a44e3e8..4cc2193 100644 --- a/tests/authorization/test_owner_privileges.py +++ b/tests/authorization/test_owner_privileges.py @@ -127,14 +127,16 @@ class TestOwnerPrivileges(SentryCacheTestSuite): @pytest.mark.execute_serially @SentryCacheTestSuite.with_args( impalad_args="--server_name=server1 --sentry_config={0} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " - .format(SENTRY_CONFIG_FILE_OO), + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " + .format(SENTRY_CONFIG_FILE_OO), catalogd_args="--sentry_config={0} --authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " - "--sentry_catalog_polling_frequency_s={1}".format(SENTRY_CONFIG_FILE_OO, - str(SENTRY_LONG_POLLING_FREQUENCY_S)), - sentry_config=SENTRY_CONFIG_FILE_OO) + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + " --sentry_catalog_polling_frequency_s={1}" + .format(SENTRY_CONFIG_FILE_OO, str(SENTRY_LONG_POLLING_FREQUENCY_S)), + sentry_config=SENTRY_CONFIG_FILE_OO, + sentry_log_dir="{0}/test_owner_privileges_with_grant_log_poll" + .format(SENTRY_BASE_LOG_DIR)) def test_owner_privileges_with_grant_long_poll(self, vector, unique_database): self.__execute_owner_privilege_tests(TestObject(TestObject.DATABASE, "owner_priv_db", grant=True)) @@ -146,14 +148,16 @@ class TestOwnerPrivileges(SentryCacheTestSuite): @pytest.mark.execute_serially @SentryCacheTestSuite.with_args( impalad_args="--server_name=server1 --sentry_config={0} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" - .format(SENTRY_CONFIG_FILE_OO), + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + .format(SENTRY_CONFIG_FILE_OO), catalogd_args="--sentry_config={0} --sentry_catalog_polling_frequency_s={1} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" - .format(SENTRY_CONFIG_FILE_OO, str(SENTRY_POLLING_FREQUENCY_S)), - sentry_config=SENTRY_CONFIG_FILE_OO) + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + .format(SENTRY_CONFIG_FILE_OO, str(SENTRY_POLLING_FREQUENCY_S)), + sentry_config=SENTRY_CONFIG_FILE_OO, + sentry_log_dir="{0}/test_owner_privileges_with_grant" + .format(SENTRY_BASE_LOG_DIR)) def test_owner_privileges_with_grant(self, vector, unique_database): self.__execute_owner_privilege_tests(TestObject(TestObject.DATABASE, "owner_priv_db", grant=True), sentry_refresh_timeout_s=SENTRY_REFRESH_TIMEOUT_S) @@ -238,14 +242,17 @@ class TestOwnerPrivileges(SentryCacheTestSuite): @pytest.mark.execute_serially @SentryCacheTestSuite.with_args( impalad_args="--server_name=server1 --sentry_config={0} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " - .format(SENTRY_CONFIG_FILE_NO_OO), + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " + .format(SENTRY_CONFIG_FILE_NO_OO), catalogd_args="--sentry_config={0} --authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " - "--sentry_catalog_polling_frequency_s={1}".format(SENTRY_CONFIG_FILE_NO_OO, - str(SENTRY_LONG_POLLING_FREQUENCY_S)), - sentry_config=SENTRY_CONFIG_FILE_NO_OO) + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + " --sentry_catalog_polling_frequency_s={1}" + .format(SENTRY_CONFIG_FILE_NO_OO, + str(SENTRY_LONG_POLLING_FREQUENCY_S)), + sentry_config=SENTRY_CONFIG_FILE_NO_OO, + sentry_log_dir="{0}/test_owner_privileges_disabled_log_poll" + .format(SENTRY_BASE_LOG_DIR)) def test_owner_privileges_disabled_long_poll(self, vector, unique_database): self.__execute_owner_privilege_tests_no_oo(TestObject(TestObject.DATABASE, "owner_priv_db")) @@ -257,14 +264,16 @@ class TestOwnerPrivileges(SentryCacheTestSuite): @pytest.mark.execute_serially @SentryCacheTestSuite.with_args( impalad_args="--server_name=server1 --sentry_config={0} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" - .format(SENTRY_CONFIG_FILE_NO_OO), + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + .format(SENTRY_CONFIG_FILE_NO_OO), catalogd_args="--sentry_config={0} --sentry_catalog_polling_frequency_s={1} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" - .format(SENTRY_CONFIG_FILE_NO_OO, str(SENTRY_POLLING_FREQUENCY_S)), - sentry_config=SENTRY_CONFIG_FILE_NO_OO) + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + .format(SENTRY_CONFIG_FILE_NO_OO, str(SENTRY_POLLING_FREQUENCY_S)), + sentry_config=SENTRY_CONFIG_FILE_NO_OO, + sentry_log_dir="{0}/test_owner_privileges_disabled" + .format(SENTRY_BASE_LOG_DIR)) def test_owner_privileges_disabled(self, vector, unique_database): self.__execute_owner_privilege_tests_no_oo(TestObject(TestObject.DATABASE, "owner_priv_db"), sentry_refresh_timeout_s=SENTRY_REFRESH_TIMEOUT_S) @@ -311,14 +320,17 @@ class TestOwnerPrivileges(SentryCacheTestSuite): @pytest.mark.execute_serially @SentryCacheTestSuite.with_args( impalad_args="--server_name=server1 --sentry_config={0} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " - .format(SENTRY_CONFIG_FILE_OO_NOGRANT), + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + .format(SENTRY_CONFIG_FILE_OO_NOGRANT), catalogd_args="--sentry_config={0} --authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider " - "--sentry_catalog_polling_frequency_s={1}".format(SENTRY_CONFIG_FILE_OO_NOGRANT, - str(SENTRY_LONG_POLLING_FREQUENCY_S)), - sentry_config=SENTRY_CONFIG_FILE_OO_NOGRANT) + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + " --sentry_catalog_polling_frequency_s={1}" + .format(SENTRY_CONFIG_FILE_OO_NOGRANT, + str(SENTRY_LONG_POLLING_FREQUENCY_S)), + sentry_config=SENTRY_CONFIG_FILE_OO_NOGRANT, + sentry_log_dir="{0}/test_owner_privileges_without_grant_log_poll" + .format(SENTRY_BASE_LOG_DIR)) def test_owner_privileges_without_grant_long_poll(self, vector, unique_database): self.__execute_owner_privilege_tests_oo_nogrant(TestObject(TestObject.DATABASE, "owner_priv_db")) @@ -330,14 +342,17 @@ class TestOwnerPrivileges(SentryCacheTestSuite): @pytest.mark.execute_serially @SentryCacheTestSuite.with_args( impalad_args="--server_name=server1 --sentry_config={0} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" - .format(SENTRY_CONFIG_FILE_OO_NOGRANT), + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + .format(SENTRY_CONFIG_FILE_OO_NOGRANT), catalogd_args="--sentry_config={0} --sentry_catalog_polling_frequency_s={1} " - "--authorization_policy_provider_class=" - "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" - .format(SENTRY_CONFIG_FILE_OO_NOGRANT, str(SENTRY_POLLING_FREQUENCY_S)), - sentry_config=SENTRY_CONFIG_FILE_OO_NOGRANT) + "--authorization_policy_provider_class=" + "org.apache.impala.service.CustomClusterResourceAuthorizationProvider" + .format(SENTRY_CONFIG_FILE_OO_NOGRANT, + str(SENTRY_POLLING_FREQUENCY_S)), + sentry_config=SENTRY_CONFIG_FILE_OO_NOGRANT, + sentry_log_dir="{0}/test_owner_privileges_without_grant" + .format(SENTRY_BASE_LOG_DIR)) def test_owner_privileges_without_grant(self, vector, unique_database): self.__execute_owner_privilege_tests_oo_nogrant(TestObject(TestObject.DATABASE, "owner_priv_db"), sentry_refresh_timeout_s=SENTRY_REFRESH_TIMEOUT_S) http://git-wip-us.apache.org/repos/asf/impala/blob/21f521a7/tests/common/custom_cluster_test_suite.py ---------------------------------------------------------------------- diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py index 8fc24c2..4274abf 100644 --- a/tests/common/custom_cluster_test_suite.py +++ b/tests/common/custom_cluster_test_suite.py @@ -42,10 +42,11 @@ CATALOGD_ARGS = 'catalogd_args' # Additional args passed to the start-impala-cluster script. START_ARGS = 'start_args' SENTRY_CONFIG = 'sentry_config' +SENTRY_LOG_DIR = 'sentry_log_dir' # Default query options passed to the impala daemon command line. Handled separately from # other impala daemon arguments to allow merging multiple defaults into a single list. DEFAULT_QUERY_OPTIONS = 'default_query_options' -LOG_DIR = 'log_dir' +IMPALA_LOG_DIR = 'impala_log_dir' # Run with fast topic updates by default to reduce time to first query running. DEFAULT_STATESTORE_ARGS = '--statestore_update_frequency_ms=50 \ @@ -96,7 +97,8 @@ class CustomClusterTestSuite(ImpalaTestSuite): @staticmethod def with_args(impalad_args=None, statestored_args=None, catalogd_args=None, - start_args=None, sentry_config=None, default_query_options=None, log_dir=None): + start_args=None, sentry_config=None, default_query_options=None, + impala_log_dir=None, sentry_log_dir=None): """Records arguments to be passed to a cluster by adding them to the decorated method's func_dict""" def decorate(func): @@ -113,10 +115,12 @@ class CustomClusterTestSuite(ImpalaTestSuite): func.func_dict[START_ARGS] = start_args if sentry_config is not None: func.func_dict[SENTRY_CONFIG] = sentry_config + if sentry_log_dir is not None: + func.func_dict[SENTRY_LOG_DIR] = sentry_log_dir if default_query_options is not None: func.func_dict[DEFAULT_QUERY_OPTIONS] = default_query_options - if log_dir is not None: - func.func_dict[LOG_DIR] = log_dir + if impala_log_dir is not None: + func.func_dict[IMPALA_LOG_DIR] = impala_log_dir return func return decorate @@ -129,12 +133,13 @@ class CustomClusterTestSuite(ImpalaTestSuite): cluster_args.append(method.func_dict[START_ARGS]) if SENTRY_CONFIG in method.func_dict: - self._start_sentry_service(method.func_dict[SENTRY_CONFIG]) + self._start_sentry_service(method.func_dict[SENTRY_CONFIG], + method.func_dict.get(SENTRY_LOG_DIR)) # Start a clean new cluster before each test - if LOG_DIR in method.func_dict: + if IMPALA_LOG_DIR in method.func_dict: self._start_impala_cluster(cluster_args, default_query_options=method.func_dict.get(DEFAULT_QUERY_OPTIONS), - log_dir=method.func_dict[LOG_DIR]) + impala_log_dir=method.func_dict[IMPALA_LOG_DIR]) else: self._start_impala_cluster(cluster_args, default_query_options=method.func_dict.get(DEFAULT_QUERY_OPTIONS)) @@ -152,8 +157,10 @@ class CustomClusterTestSuite(ImpalaTestSuite): sleep(2) @classmethod - def _start_sentry_service(cls, sentry_service_config): + def _start_sentry_service(cls, sentry_service_config, sentry_log_dir=None): sentry_env = dict(os.environ) + if sentry_log_dir is not None: + sentry_env['SENTRY_LOG_DIR'] = sentry_log_dir sentry_env['SENTRY_SERVICE_CONFIG'] = sentry_service_config call = subprocess.Popen( ['/bin/bash', '-c', os.path.join(IMPALA_HOME, @@ -164,18 +171,18 @@ class CustomClusterTestSuite(ImpalaTestSuite): raise RuntimeError("unable to start sentry") @classmethod - def _start_impala_cluster(cls, options, log_dir=os.getenv('LOG_DIR', "/tmp/"), + def _start_impala_cluster(cls, options, impala_log_dir=os.getenv('LOG_DIR', "/tmp/"), cluster_size=CLUSTER_SIZE, num_coordinators=NUM_COORDINATORS, use_exclusive_coordinators=False, log_level=1, expected_num_executors=CLUSTER_SIZE, default_query_options=None): - cls.impala_log_dir = log_dir + cls.impala_log_dir = impala_log_dir # We ignore TEST_START_CLUSTER_ARGS here. Custom cluster tests specifically test that # certain custom startup arguments work and we want to keep them independent of dev # environments. cmd = [os.path.join(IMPALA_HOME, 'bin/start-impala-cluster.py'), '--cluster_size=%d' % cluster_size, '--num_coordinators=%d' % num_coordinators, - '--log_dir=%s' % log_dir, + '--log_dir=%s' % impala_log_dir, '--log_level=%s' % log_level] if use_exclusive_coordinators: http://git-wip-us.apache.org/repos/asf/impala/blob/21f521a7/tests/custom_cluster/test_redaction.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_redaction.py b/tests/custom_cluster/test_redaction.py index 9cdb71d..7789e06 100644 --- a/tests/custom_cluster/test_redaction.py +++ b/tests/custom_cluster/test_redaction.py @@ -97,7 +97,7 @@ class TestRedaction(CustomClusterTestSuite, unittest.TestCase): -redaction_rules_file=%s -vmodule=%s'""" % (self.audit_dir, self.profile_dir, self.rules_file, vmodule)], - log_dir=self.log_dir, + impala_log_dir=self.log_dir, log_level=log_level) self.client = self.create_impala_client()
