Repository: ambari Updated Branches: refs/heads/trunk 40ffee9c8 -> 4c5af44c8
AMBARI-8482. HBase service components should indicate security state (rlevas) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4c5af44c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4c5af44c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4c5af44c Branch: refs/heads/trunk Commit: 4c5af44c8481453f6a987302a514e77b80eee0fe Parents: 40ffee9 Author: Robert Levas <rle...@hortonworks.com> Authored: Tue Jan 6 17:53:15 2015 -0500 Committer: Robert Levas <rle...@hortonworks.com> Committed: Tue Jan 6 17:53:15 2015 -0500 ---------------------------------------------------------------------- .../0.96.0.2.0/package/scripts/hbase_master.py | 52 +++++++++- .../package/scripts/hbase_regionserver.py | 54 +++++++++- .../0.96.0.2.0/package/scripts/status_params.py | 10 ++ .../stacks/HDP/2.2/services/HBASE/kerberos.json | 3 - .../stacks/2.0.6/HBASE/test_hbase_master.py | 101 ++++++++++++++++++ .../2.0.6/HBASE/test_hbase_regionserver.py | 102 +++++++++++++++++++ 6 files changed, 316 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4c5af44c/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py index c0e84b4..ce65507 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py @@ -20,7 +20,9 @@ limitations under the License. import sys from resource_management import * - +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML from hbase import hbase from hbase_service import hbase_service from hbase_decommission import hbase_decommission @@ -65,6 +67,54 @@ class HbaseMaster(Script): pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid") check_process_status(pid_file) + def security_status(self, env): + import status_params + + env.set_params(status_params) + + props_value_check = {"hbase.security.authentication" : "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ['hbase.master.keytab.file', + 'hbase.master.kerberos.principal'] + props_read_check = ['hbase.master.keytab.file'] + hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, + props_read_check) + + hbase_expectations = {} + hbase_expectations.update(hbase_site_expectations) + + security_params = get_params_from_filesystem(status_params.hbase_conf_dir, + {'hbase-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, hbase_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'hbase-site' not in security_params + or 'hbase.master.keytab.file' not in security_params['hbase-site'] + or 'hbase.master.kerberos.principal' not in security_params['hbase-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.master.keytab.file'], + security_params['hbase-site']['hbase.master.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + def decommission(self, env): import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/4c5af44c/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py index ea8e3d4..010d045 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py @@ -20,7 +20,9 @@ limitations under the License. import sys from resource_management import * - +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML from hbase import hbase from hbase_service import hbase_service import upgrade @@ -69,7 +71,55 @@ class HbaseRegionServer(Script): env.set_params(status_params) pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid") check_process_status(pid_file) - + + def security_status(self, env): + import status_params + + env.set_params(status_params) + + props_value_check = {"hbase.security.authentication" : "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ['hbase.regionserver.keytab.file', + 'hbase.regionserver.kerberos.principal'] + props_read_check = ['hbase.regionserver.keytab.file'] + hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, + props_read_check) + + hbase_expectations = {} + hbase_expectations.update(hbase_site_expectations) + + security_params = get_params_from_filesystem(status_params.hbase_conf_dir, + {'hbase-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, hbase_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'hbase-site' not in security_params + or 'hbase.regionserver.keytab.file' not in security_params['hbase-site'] + or 'hbase.regionserver.kerberos.principal' not in security_params['hbase-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.regionserver.keytab.file'], + security_params['hbase-site']['hbase.regionserver.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + def decommission(self, env): print "Decommission not yet implemented!" http://git-wip-us.apache.org/repos/asf/ambari/blob/4c5af44c/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py index 850ec8b..1487ae0 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py @@ -24,3 +24,13 @@ config = Script.get_config() pid_dir = config['configurations']['hbase-env']['hbase_pid_dir'] hbase_user = config['configurations']['hbase-env']['hbase_user'] + +# Security related/required params +hostname = config['hostname'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +tmp_dir = Script.get_tmp_dir() + + +hbase_conf_dir_prefix = "/etc/hbase" +hbase_conf_dir = format("{hbase_conf_dir_prefix}/conf") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4c5af44c/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/kerberos.json index 17d0c2f..6dd3e84 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/kerberos.json @@ -91,9 +91,6 @@ } } ] - }, - { - "name": "HBASE_CLIENT" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/4c5af44c/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py index 5420bc7..0d04acf 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py @@ -541,3 +541,104 @@ class TestHBaseMaster(RMFTestCase): self.assertNoMoreResources() + @patch("resource_management.libraries.functions.security_commons.build_expectations") + @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") + @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties") + @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor") + @patch("resource_management.libraries.script.Script.put_structured_out") + def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock): + # Test that function works when is called with correct parameters + import status_params + + security_params = {} + security_params['hbase-site'] = {} + security_params['hbase-site']['hbase.master.kerberos.principal'] = '/path/to/hbase_keytab' + security_params['hbase-site']['hbase.master.keytab.file'] = 'hbase_principal' + + result_issues = [] + props_value_check = {"hbase.security.authentication": "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ["hbase.master.keytab.file", + "hbase.master.kerberos.principal"] + + props_read_check = ["hbase.master.keytab.file"] + + get_params_mock.return_value = security_params + validate_security_config_mock.return_value = result_issues + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py", + classname = "HbaseMaster", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + build_exp_mock.assert_called_with('hbase-site', props_value_check, props_empty_check, props_read_check) + put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) + cached_kinit_executor_mock.called_with(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.master.keytab.file'], + security_params['hbase-site']['hbase.master.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + + # Testing that the exception throw by cached_executor is caught + cached_kinit_executor_mock.reset_mock() + cached_kinit_executor_mock.side_effect = Exception("Invalid command") + + try: + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py", + classname = "HbaseMaster", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + except: + self.assertTrue(True) + + # Testing with a security_params which doesn't contains hbase-site + empty_security_params = {} + cached_kinit_executor_mock.reset_mock() + get_params_mock.reset_mock() + put_structured_out_mock.reset_mock() + get_params_mock.return_value = empty_security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py", + classname = "HbaseMaster", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."}) + + # Testing with not empty result_issues + result_issues_with_params = {} + result_issues_with_params['hbase-site']="Something bad happened" + + validate_security_config_mock.reset_mock() + get_params_mock.reset_mock() + validate_security_config_mock.return_value = result_issues_with_params + get_params_mock.return_value = security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py", + classname = "HbaseMaster", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + + # Testing with security_enable = false + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py", + classname = "HbaseMaster", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4c5af44c/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py index 10bd583..bdd5954 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py @@ -467,3 +467,105 @@ class TestHbaseRegionServer(RMFTestCase): user = 'hbase') self.assertNoMoreResources() + + @patch("resource_management.libraries.functions.security_commons.build_expectations") + @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") + @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties") + @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor") + @patch("resource_management.libraries.script.Script.put_structured_out") + def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock): + # Test that function works when is called with correct parameters + import status_params + + security_params = {} + security_params['hbase-site'] = {} + security_params['hbase-site']['hbase.regionserver.keytab.file'] = '/path/to/hbase_keytab' + security_params['hbase-site']['hbase.regionserver.kerberos.principal'] = 'hbase_principal' + + result_issues = [] + props_value_check = {"hbase.security.authentication": "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ["hbase.regionserver.keytab.file", + "hbase.regionserver.kerberos.principal"] + + props_read_check = ["hbase.regionserver.keytab.file"] + + get_params_mock.return_value = security_params + validate_security_config_mock.return_value = result_issues + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py", + classname = "HbaseRegionServer", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + build_exp_mock.assert_called_with('hbase-site', props_value_check, props_empty_check, props_read_check) + put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) + cached_kinit_executor_mock.called_with(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.regionserver.keytab.file'], + security_params['hbase-site']['hbase.regionserver.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + + # Testing that the exception throw by cached_executor is caught + cached_kinit_executor_mock.reset_mock() + cached_kinit_executor_mock.side_effect = Exception("Invalid command") + + try: + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py", + classname = "HbaseRegionServer", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + except: + self.assertTrue(True) + + # Testing with a security_params which doesn't contains hbase-site + empty_security_params = {} + cached_kinit_executor_mock.reset_mock() + get_params_mock.reset_mock() + put_structured_out_mock.reset_mock() + get_params_mock.return_value = empty_security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py", + classname = "HbaseRegionServer", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."}) + + # Testing with not empty result_issues + result_issues_with_params = {} + result_issues_with_params['hbase-site']="Something bad happened" + + validate_security_config_mock.reset_mock() + get_params_mock.reset_mock() + validate_security_config_mock.return_value = result_issues_with_params + get_params_mock.return_value = security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py", + classname = "HbaseRegionServer", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + + # Testing with security_enable = false + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py", + classname = "HbaseRegionServer", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file