AMBARI-9804. Agent is spamming logs with exceptions (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b8cd75d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b8cd75d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b8cd75d Branch: refs/heads/trunk Commit: 4b8cd75d6e0674b73d7c1781a4994071f164dd4f Parents: 3cdc032 Author: Robert Levas <[email protected]> Authored: Fri Feb 27 09:02:20 2015 -0500 Committer: Robert Levas <[email protected]> Committed: Fri Feb 27 09:02:31 2015 -0500 ---------------------------------------------------------------------- .../libraries/script/script.py | 7 +- .../0.96.0.2.0/package/scripts/hbase_master.py | 84 +++++++------- .../package/scripts/hbase_regionserver.py | 84 +++++++------- .../HDFS/2.1.0.2.0/package/scripts/datanode.py | 57 +++++----- .../2.1.0.2.0/package/scripts/hdfs_client.py | 26 +++-- .../2.1.0.2.0/package/scripts/journalnode.py | 56 +++++----- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 55 ++++----- .../HDFS/2.1.0.2.0/package/scripts/snamenode.py | 59 +++++----- .../2.1.0.2.0/package/scripts/zkfc_slave.py | 27 ++--- .../scripts/application_timeline_server.py | 112 ++++++++++--------- .../2.1.0.2.0/package/scripts/historyserver.py | 98 ++++++++-------- .../2.1.0.2.0/package/scripts/nodemanager.py | 111 +++++++++--------- .../package/scripts/resourcemanager.py | 112 +++++++++---------- .../stacks/2.0.6/HBASE/test_hbase_master.py | 2 +- .../2.0.6/HBASE/test_hbase_regionserver.py | 4 +- .../python/stacks/2.0.6/HDFS/test_datanode.py | 107 ++++++++++++++++++ .../stacks/2.0.6/HDFS/test_hdfs_client.py | 97 ++++++++++++++++ .../stacks/2.0.6/HDFS/test_journalnode.py | 111 +++++++++++++++++- .../python/stacks/2.0.6/HDFS/test_namenode.py | 111 ++++++++++++++++++ .../python/stacks/2.0.6/HDFS/test_snamenode.py | 111 ++++++++++++++++++ .../test/python/stacks/2.0.6/HDFS/test_zkfc.py | 98 ++++++++++++++++ .../stacks/2.0.6/YARN/test_nodemanager.py | 3 +- .../stacks/2.0.6/YARN/test_resourcemanager.py | 3 +- .../python/stacks/2.0.6/configs/default.json | 2 + .../stacks/2.1/YARN/test_apptimelineserver.py | 109 ++++++++++++++++++ 25 files changed, 1211 insertions(+), 435 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 9b3611b..00580c0 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -110,7 +110,12 @@ class Script(object): # version is only set in a specific way and should not be carried if "version" in Script.structuredOut: del Script.structuredOut["version"] - + # reset security issues and errors found on previous runs + if "securityIssuesFound" in Script.structuredOut: + del Script.structuredOut["securityIssuesFound"] + if "securityStateErrorInfo" in Script.structuredOut: + del Script.structuredOut["securityStateErrorInfo"] + def put_structured_out(self, sout): curr_content = Script.structuredOut.copy() Script.structuredOut.update(sout) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py index 6059a1c..e87efbf 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py @@ -75,48 +75,50 @@ class HbaseMaster(Script): import status_params env.set_params(status_params) - - props_value_check = {"hbase.security.authentication" : "kerberos", - "hbase.security.authorization": "true"} - props_empty_check = ['hbase.master.keytab.file', - 'hbase.master.kerberos.principal'] - props_read_check = ['hbase.master.keytab.file'] - hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, - props_read_check) - - hbase_expectations = {} - hbase_expectations.update(hbase_site_expectations) - - security_params = get_params_from_filesystem(status_params.hbase_conf_dir, - {'hbase-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, hbase_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'hbase-site' not in security_params - or 'hbase.master.keytab.file' not in security_params['hbase-site'] - or 'hbase.master.kerberos.principal' not in security_params['hbase-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.hbase_user, - security_params['hbase-site']['hbase.master.keytab.file'], - security_params['hbase-site']['hbase.master.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if status_params.security_enabled: + props_value_check = {"hbase.security.authentication" : "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ['hbase.master.keytab.file', + 'hbase.master.kerberos.principal'] + props_read_check = ['hbase.master.keytab.file'] + hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, + props_read_check) + + hbase_expectations = {} + hbase_expectations.update(hbase_site_expectations) + + security_params = get_params_from_filesystem(status_params.hbase_conf_dir, + {'hbase-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, hbase_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'hbase-site' not in security_params + or 'hbase.master.keytab.file' not in security_params['hbase-site'] + or 'hbase.master.kerberos.principal' not in security_params['hbase-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.master.keytab.file'], + security_params['hbase-site']['hbase.master.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) def decommission(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py index 0f0f539..fc2b677 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py @@ -80,48 +80,50 @@ class HbaseRegionServer(Script): import status_params env.set_params(status_params) - - props_value_check = {"hbase.security.authentication" : "kerberos", - "hbase.security.authorization": "true"} - props_empty_check = ['hbase.regionserver.keytab.file', - 'hbase.regionserver.kerberos.principal'] - props_read_check = ['hbase.regionserver.keytab.file'] - hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, - props_read_check) - - hbase_expectations = {} - hbase_expectations.update(hbase_site_expectations) - - security_params = get_params_from_filesystem(status_params.hbase_conf_dir, - {'hbase-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, hbase_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'hbase-site' not in security_params - or 'hbase.regionserver.keytab.file' not in security_params['hbase-site'] - or 'hbase.regionserver.kerberos.principal' not in security_params['hbase-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.hbase_user, - security_params['hbase-site']['hbase.regionserver.keytab.file'], - security_params['hbase-site']['hbase.regionserver.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if status_params.security_enabled: + props_value_check = {"hbase.security.authentication" : "kerberos", + "hbase.security.authorization": "true"} + props_empty_check = ['hbase.regionserver.keytab.file', + 'hbase.regionserver.kerberos.principal'] + props_read_check = ['hbase.regionserver.keytab.file'] + hbase_site_expectations = build_expectations('hbase-site', props_value_check, props_empty_check, + props_read_check) + + hbase_expectations = {} + hbase_expectations.update(hbase_site_expectations) + + security_params = get_params_from_filesystem(status_params.hbase_conf_dir, + {'hbase-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, hbase_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'hbase-site' not in security_params + or 'hbase.regionserver.keytab.file' not in security_params['hbase-site'] + or 'hbase.regionserver.kerberos.principal' not in security_params['hbase-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hbase_user, + security_params['hbase-site']['hbase.regionserver.keytab.file'], + security_params['hbase-site']['hbase.regionserver.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) def decommission(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py index db07932..7ebdeb3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py @@ -113,33 +113,38 @@ class DataNode(Script): security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML, 'hdfs-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, hdfs_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ('hdfs-site' not in security_params or - 'dfs.datanode.keytab.file' not in security_params['hdfs-site'] or - 'dfs.datanode.kerberos.principal' not in security_params['hdfs-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.hdfs_user, - security_params['hdfs-site']['dfs.datanode.keytab.file'], - security_params['hdfs-site']['dfs.datanode.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': + result_issues = validate_security_config_properties(security_params, hdfs_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ('hdfs-site' not in security_params or + 'dfs.datanode.keytab.file' not in security_params['hdfs-site'] or + 'dfs.datanode.kerberos.principal' not in security_params['hdfs-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['dfs.datanode.keytab.file'], + security_params['hdfs-site']['dfs.datanode.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py index fa890ad..df9cdd7 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py @@ -63,19 +63,21 @@ class HdfsClient(Script): def security_status(self, env): import status_params env.set_params(status_params) - if status_params.security_enabled: - props_value_check = {"hadoop.security.authentication": "kerberos", - "hadoop.security.authorization": "true"} - props_empty_check = ["hadoop.security.auth_to_local"] - props_read_check = None - core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, - props_read_check) - - hdfs_expectations ={} - hdfs_expectations.update(core_site_expectations) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + + props_value_check = {"hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true"} + props_empty_check = ["hadoop.security.auth_to_local"] + props_read_check = None + core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, + props_read_check) + hdfs_expectations ={} + hdfs_expectations.update(core_site_expectations) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML}) + + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully if status_params.hdfs_user_principal or status_params.hdfs_user_keytab: http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py index 505d0b7..7f9243b 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py @@ -124,33 +124,37 @@ class JournalNode(Script): security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, hdfs_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ('hdfs-site' not in security_params or - 'dfs.journalnode.kerberos.keytab.file' not in security_params['hdfs-site'] or - 'dfs.journalnode.kerberos.principal' not in security_params['hdfs-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.hdfs_user, - security_params['hdfs-site']['dfs.journalnode.kerberos.keytab.file'], - security_params['hdfs-site']['dfs.journalnode.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': + result_issues = validate_security_config_properties(security_params, hdfs_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ('hdfs-site' not in security_params or + 'dfs.journalnode.kerberos.keytab.file' not in security_params['hdfs-site'] or + 'dfs.journalnode.kerberos.principal' not in security_params['hdfs-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['dfs.journalnode.kerberos.keytab.file'], + security_params['hdfs-site']['dfs.journalnode.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 3dfdb88..bc10adf 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -134,33 +134,36 @@ class NameNode(Script): security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML, 'hdfs-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, hdfs_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'hdfs-site' not in security_params - or 'dfs.namenode.keytab.file' not in security_params['hdfs-site'] - or 'dfs.namenode.kerberos.principal' not in security_params['hdfs-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.hdfs_user, - security_params['hdfs-site']['dfs.namenode.keytab.file'], - security_params['hdfs-site']['dfs.namenode.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': + result_issues = validate_security_config_properties(security_params, hdfs_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'hdfs-site' not in security_params + or 'dfs.namenode.keytab.file' not in security_params['hdfs-site'] + or 'dfs.namenode.kerberos.principal' not in security_params['hdfs-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + cached_kinit_executor(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['dfs.namenode.keytab.file'], + security_params['hdfs-site']['dfs.namenode.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py index 49249b6..d5b85bd 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py @@ -94,34 +94,39 @@ class SNameNode(Script): security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML, 'hdfs-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, hdfs_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ('hdfs-site' not in security_params or - 'dfs.secondary.namenode.keytab.file' not in security_params['hdfs-site'] or - 'dfs.secondary.namenode.kerberos.principal' not in security_params['hdfs-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.hdfs_user, - security_params['hdfs-site']['dfs.secondary.namenode.keytab.file'], - security_params['hdfs-site'][ - 'dfs.secondary.namenode.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': + result_issues = validate_security_config_properties(security_params, hdfs_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ('hdfs-site' not in security_params or + 'dfs.secondary.namenode.keytab.file' not in security_params['hdfs-site'] or + 'dfs.secondary.namenode.kerberos.principal' not in security_params['hdfs-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['dfs.secondary.namenode.keytab.file'], + security_params['hdfs-site'][ + 'dfs.secondary.namenode.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index c5b0b1b..42dc7d3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -80,20 +80,21 @@ class ZkfcSlave(Script): import status_params env.set_params(status_params) - if status_params.security_enabled: - props_value_check = {"hadoop.security.authentication": "kerberos", - "hadoop.security.authorization": "true"} - props_empty_check = ["hadoop.security.auth_to_local"] - props_read_check = None - core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, - props_read_check) - - hdfs_expectations = {} - hdfs_expectations.update(core_site_expectations) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + + props_value_check = {"hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true"} + props_empty_check = ["hadoop.security.auth_to_local"] + props_read_check = None + core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, + props_read_check) + hdfs_expectations = {} + hdfs_expectations.update(core_site_expectations) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, {'core-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, hdfs_expectations) + result_issues = validate_security_config_properties(security_params, hdfs_expectations) + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': if not result_issues: # If all validations passed successfully if status_params.hdfs_user_principal or status_params.hdfs_user_keytab: try: http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py index 8971bf5..b41eadf 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py @@ -72,62 +72,64 @@ class ApplicationTimelineServer(Script): def security_status(self, env): import status_params env.set_params(status_params) - - props_value_check = {"yarn.timeline-service.enabled": "true", - "yarn.timeline-service.http-authentication.type": "kerberos", - "yarn.acl.enable": "true"} - props_empty_check = ["yarn.timeline-service.principal", - "yarn.timeline-service.keytab", - "yarn.timeline-service.http-authentication.kerberos.principal", - "yarn.timeline-service.http-authentication.kerberos.keytab"] - - props_read_check = ["yarn.timeline-service.keytab", - "yarn.timeline-service.http-authentication.kerberos.keytab"] - yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, - props_read_check) - - yarn_expectations ={} - yarn_expectations.update(yarn_site_props) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'yarn-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, yarn_expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'yarn-site' not in security_params - or 'yarn.timeline-service.keytab' not in security_params['yarn-site'] - or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \ - or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \ - or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']: - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.timeline-service.keytab'], - security_params['yarn-site']['yarn.timeline-service.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'], - security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if status_params.security_enabled: + props_value_check = {"yarn.timeline-service.enabled": "true", + "yarn.timeline-service.http-authentication.type": "kerberos", + "yarn.acl.enable": "true"} + props_empty_check = ["yarn.timeline-service.principal", + "yarn.timeline-service.keytab", + "yarn.timeline-service.http-authentication.kerberos.principal", + "yarn.timeline-service.http-authentication.kerberos.keytab"] + + props_read_check = ["yarn.timeline-service.keytab", + "yarn.timeline-service.http-authentication.kerberos.keytab"] + yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, + props_read_check) + + yarn_expectations ={} + yarn_expectations.update(yarn_site_props) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'yarn-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, yarn_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'yarn-site' not in security_params + or 'yarn.timeline-service.keytab' not in security_params['yarn-site'] + or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \ + or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \ + or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']: + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.timeline-service.keytab'], + security_params['yarn-site']['yarn.timeline-service.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'], + security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py index 45e4d9b..9a43efa 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py @@ -73,55 +73,57 @@ class HistoryServer(Script): def security_status(self, env): import status_params env.set_params(status_params) - - expectations = {} - expectations.update(build_expectations('mapred-site', - None, - [ - 'mapreduce.jobhistory.keytab', - 'mapreduce.jobhistory.principal', - 'mapreduce.jobhistory.webapp.spnego-keytab-file', - 'mapreduce.jobhistory.webapp.spnego-principal' - ], - None)) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'mapred-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, expectations) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'mapred-site' not in security_params or - 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or - 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or - 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or - 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']): - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal not set."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.mapred_user, - security_params['mapred-site']['mapreduce.jobhistory.keytab'], - security_params['mapred-site']['mapreduce.jobhistory.principal'], - status_params.hostname, - status_params.tmp_dir) - cached_kinit_executor(status_params.kinit_path_local, - status_params.mapred_user, - security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'], - security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'], - status_params.hostname, - status_params.tmp_dir) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if status_params.security_enabled: + expectations = {} + expectations.update(build_expectations('mapred-site', + None, + [ + 'mapreduce.jobhistory.keytab', + 'mapreduce.jobhistory.principal', + 'mapreduce.jobhistory.webapp.spnego-keytab-file', + 'mapreduce.jobhistory.webapp.spnego-principal' + ], + None)) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'mapred-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'mapred-site' not in security_params or + 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or + 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or + 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or + 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal not set."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.mapred_user, + security_params['mapred-site']['mapreduce.jobhistory.keytab'], + security_params['mapred-site']['mapreduce.jobhistory.principal'], + status_params.hostname, + status_params.tmp_dir) + cached_kinit_executor(status_params.kinit_path_local, + status_params.mapred_user, + security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'], + security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py index 53395ca..1f2300b 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py @@ -79,62 +79,63 @@ class Nodemanager(Script): def security_status(self, env): import status_params env.set_params(status_params) - - props_value_check = {"yarn.timeline-service.enabled": "true", - "yarn.timeline-service.http-authentication.type": "kerberos", - "yarn.acl.enable": "true"} - props_empty_check = ["yarn.nodemanager.principal", - "yarn.nodemanager.keytab", - "yarn.nodemanager.webapp.spnego-principal", - "yarn.nodemanager.webapp.spnego-keytab-file"] - - props_read_check = ["yarn.nodemanager.keytab", - "yarn.nodemanager.webapp.spnego-keytab-file"] - yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, - props_read_check) - - yarn_expectations ={} - yarn_expectations.update(yarn_site_props) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'yarn-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, yarn_site_props) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'yarn-site' not in security_params - or 'yarn.nodemanager.keytab' not in security_params['yarn-site'] - or 'yarn.nodemanager.principal' not in security_params['yarn-site']) \ - or 'yarn.nodemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ - or 'yarn.nodemanager.webapp.spnego-principal' not in security_params['yarn-site']: - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.nodemanager.keytab'], - security_params['yarn-site']['yarn.nodemanager.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'], - security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if status_params.security_enabled: + props_value_check = {"yarn.timeline-service.http-authentication.type": "kerberos", + "yarn.acl.enable": "true"} + props_empty_check = ["yarn.nodemanager.principal", + "yarn.nodemanager.keytab", + "yarn.nodemanager.webapp.spnego-principal", + "yarn.nodemanager.webapp.spnego-keytab-file"] + + props_read_check = ["yarn.nodemanager.keytab", + "yarn.nodemanager.webapp.spnego-keytab-file"] + yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, + props_read_check) + + yarn_expectations ={} + yarn_expectations.update(yarn_site_props) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'yarn-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, yarn_site_props) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'yarn-site' not in security_params + or 'yarn.nodemanager.keytab' not in security_params['yarn-site'] + or 'yarn.nodemanager.principal' not in security_params['yarn-site']) \ + or 'yarn.nodemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ + or 'yarn.nodemanager.webapp.spnego-principal' not in security_params['yarn-site']: + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.nodemanager.keytab'], + security_params['yarn-site']['yarn.nodemanager.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'], + security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index 6830a34..167d3b8 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -79,65 +79,65 @@ class Resourcemanager(Script): def security_status(self, env): import status_params env.set_params(status_params) - - props_value_check = {"yarn.timeline-service.enabled": "true", - "yarn.timeline-service.http-authentication.type": "kerberos", - "yarn.acl.enable": "true"} - props_empty_check = ["yarn.resourcemanager.principal", - "yarn.resourcemanager.keytab", - "yarn.resourcemanager.webapp.spnego-principal", - "yarn.resourcemanager.webapp.spnego-keytab-file"] - - props_read_check = ["yarn.resourcemanager.keytab", - "yarn.resourcemanager.webapp.spnego-keytab-file"] - yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, - props_read_check) - - yarn_expectations ={} - yarn_expectations.update(yarn_site_props) - - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - {'yarn-site.xml': FILE_TYPE_XML}) - result_issues = validate_security_config_properties(security_params, yarn_site_props) - if not result_issues: # If all validations passed successfully - try: - # Double check the dict before calling execute - if ( 'yarn-site' not in security_params - or 'yarn.resourcemanager.keytab' not in security_params['yarn-site'] - or 'yarn.resourcemanager.principal' not in security_params['yarn-site']) \ - or 'yarn.resourcemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ - or 'yarn.resourcemanager.webapp.spnego-principal' not in security_params['yarn-site']: - self.put_structured_out({"securityState": "UNSECURED"}) - self.put_structured_out( - {"securityIssuesFound": "Keytab file or principal are not set property."}) - return - - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.resourcemanager.keytab'], - security_params['yarn-site']['yarn.resourcemanager.principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - cached_kinit_executor(status_params.kinit_path_local, - status_params.yarn_user, - security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'], - security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'], - status_params.hostname, - status_params.tmp_dir, - 30) - self.put_structured_out({"securityState": "SECURED_KERBEROS"}) - except Exception as e: - self.put_structured_out({"securityState": "ERROR"}) - self.put_structured_out({"securityStateErrorInfo": str(e)}) + if status_params.security_enabled: + props_value_check = {"yarn.timeline-service.http-authentication.type": "kerberos", + "yarn.acl.enable": "true"} + props_empty_check = ["yarn.resourcemanager.principal", + "yarn.resourcemanager.keytab", + "yarn.resourcemanager.webapp.spnego-principal", + "yarn.resourcemanager.webapp.spnego-keytab-file"] + + props_read_check = ["yarn.resourcemanager.keytab", + "yarn.resourcemanager.webapp.spnego-keytab-file"] + yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check, + props_read_check) + + yarn_expectations ={} + yarn_expectations.update(yarn_site_props) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'yarn-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, yarn_site_props) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'yarn-site' not in security_params + or 'yarn.resourcemanager.keytab' not in security_params['yarn-site'] + or 'yarn.resourcemanager.principal' not in security_params['yarn-site']) \ + or 'yarn.resourcemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \ + or 'yarn.resourcemanager.webapp.spnego-principal' not in security_params['yarn-site']: + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.resourcemanager.keytab'], + security_params['yarn-site']['yarn.resourcemanager.principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + cached_kinit_executor(status_params.kinit_path_local, + status_params.yarn_user, + security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'], + security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'], + status_params.hostname, + status_params.tmp_dir, + 30) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = [] - for cf in result_issues: - issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) - self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) - def refreshqueues(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py index 054421f..0bb1c99 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py @@ -646,7 +646,7 @@ class TestHBaseMaster(RMFTestCase): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py", classname = "HbaseMaster", command = "security_status", - config_file="secured.json", + config_file="default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py index c5ecd39..04a272b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py @@ -584,8 +584,8 @@ class TestHbaseRegionServer(RMFTestCase): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py", classname = "HbaseRegionServer", command = "security_status", - config_file="secured.json", + config_file="default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py index 7c45583..b3aa1a4 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py @@ -516,4 +516,111 @@ class TestDatanode(RMFTestCase): self.assertTrue(process_mock.called) self.assertEqual(process_mock.call_count,12) + @patch("resource_management.libraries.functions.security_commons.build_expectations") + @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") + @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties") + @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor") + @patch("resource_management.libraries.script.Script.put_structured_out") + def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock): + # Test that function works when is called with correct parameters + import status_params + + security_params = {} + security_params['core-site'] = {} + security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + security_params['hdfs-site'] = {} + security_params['hdfs-site']['dfs.datanode.keytab.file'] = 'path/to/datanode/keytab/file' + security_params['hdfs-site']['dfs.datanode.kerberos.principal'] = 'datanode_principal' + + props_value_check = None + props_empty_check = ['dfs.datanode.keytab.file', + 'dfs.datanode.kerberos.principal'] + props_read_check = ['dfs.datanode.keytab.file'] + + result_issues = [] + + get_params_mock.return_value = security_params + validate_security_config_mock.return_value = result_issues + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py", + classname = "DataNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + build_exp_mock.assert_called_with('hdfs-site', props_value_check, props_empty_check, props_read_check) + put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) + cached_kinit_executor_mock.called_with(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['dfs.datanode.keytab.file'], + security_params['hdfs-site']['dfs.datanode.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + + # Testing when hadoop.security.authentication is simple + security_params['core-site']['hadoop.security.authentication'] = 'simple' + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py", + classname = "DataNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + + # Testing that the exception throw by cached_executor is caught + cached_kinit_executor_mock.reset_mock() + cached_kinit_executor_mock.side_effect = Exception("Invalid command") + + try: + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py", + classname = "DataNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + except: + self.assertTrue(True) + + # Testing with a security_params which doesn't contains hdfs-site + empty_security_params = {} + empty_security_params['core-site'] = {} + empty_security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + cached_kinit_executor_mock.reset_mock() + get_params_mock.reset_mock() + put_structured_out_mock.reset_mock() + get_params_mock.return_value = empty_security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py", + classname = "DataNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."}) + + # Testing with not empty result_issues + result_issues_with_params = {} + result_issues_with_params['hdfs-site']="Something bad happened" + + validate_security_config_mock.reset_mock() + get_params_mock.reset_mock() + validate_security_config_mock.return_value = result_issues_with_params + get_params_mock.return_value = security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py", + classname = "DataNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py index 005fc0c..0868d08 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py @@ -78,3 +78,100 @@ class Test(RMFTestCase): self.assertResourceCalled("Execute", "hdp-select set hadoop-client 2.2.1.0-2067") # for now, it's enough that hdp-select is confirmed + + @patch("resource_management.libraries.functions.security_commons.build_expectations") + @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") + @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties") + @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor") + @patch("resource_management.libraries.script.Script.put_structured_out") + def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock): + # Test that function works when is called with correct parameters + import status_params + + security_params = {} + security_params['core-site'] = {} + security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + + props_value_check = {"hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true"} + props_empty_check = ["hadoop.security.auth_to_local"] + props_read_check = None + + result_issues = [] + + get_params_mock.return_value = security_params + validate_security_config_mock.return_value = result_issues + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hdfs_client.py", + classname = "HdfsClient", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + build_exp_mock.assert_called_with('core-site', props_value_check, props_empty_check, props_read_check) + put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) + cached_kinit_executor_mock.called_with(status_params.kinit_path_local, + status_params.hdfs_user, + status_params.hdfs_user_keytab, + status_params.hdfs_user_principal, + status_params.hostname, + status_params.tmp_dir) + + # Testing that the exception throw by cached_executor is caught + cached_kinit_executor_mock.reset_mock() + cached_kinit_executor_mock.side_effect = Exception("Invalid command") + + try: + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hdfs_client.py", + classname = "HdfsClient", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + except: + self.assertTrue(True) + + # Testing when hadoop.security.authentication is simple + security_params['core-site']['hadoop.security.authentication'] = 'simple' + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hdfs_client.py", + classname = "HdfsClient", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + + # Testing with not empty result_issues + result_issues_with_params = {} + result_issues_with_params['hdfs-site']="Something bad happened" + + validate_security_config_mock.reset_mock() + get_params_mock.reset_mock() + validate_security_config_mock.return_value = result_issues_with_params + get_params_mock.return_value = security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hdfs_client.py", + classname = "HdfsClient", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + + # Testing with empty hdfs_user_principal and hdfs_user_keytab + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hdfs_client.py", + classname = "HdfsClient", + command = "security_status", + config_file="default.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b8cd75d/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py index ce20a5c..abaf89a 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py @@ -323,4 +323,113 @@ class TestJournalnode(RMFTestCase): self.fail("Expected a failure since the JMX JSON for JournalTransactionInfo was missing") except: - pass \ No newline at end of file + pass + + @patch("resource_management.libraries.functions.security_commons.build_expectations") + @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") + @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties") + @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor") + @patch("resource_management.libraries.script.Script.put_structured_out") + def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock): + # Test that function works when is called with correct parameters + import status_params + + security_params = {} + security_params['core-site'] = {} + security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + security_params['hdfs-site'] = {} + security_params['hdfs-site']['dfs.journalnode.kerberos.keytab.file'] = 'path/to/journalnode/keytab/file' + security_params['hdfs-site']['dfs.journalnode.kerberos.principal'] = 'journalnode_principal' + + props_value_check = None + props_empty_check = ['dfs.journalnode.keytab.file', + 'dfs.journalnode.kerberos.principal'] + props_read_check = ['dfs.journalnode.keytab.file'] + + result_issues = [] + + get_params_mock.return_value = security_params + validate_security_config_mock.return_value = result_issues + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py", + classname = "JournalNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + build_exp_mock.assert_called_with('hdfs-site', props_value_check, props_empty_check, props_read_check) + put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) + cached_kinit_executor_mock.called_with(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['dfs.journalnode.kerberos.keytab.file'], + security_params['hdfs-site']['dfs.journalnode.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + + # Testing when hadoop.security.authentication is simple + security_params['core-site']['hadoop.security.authentication'] = 'simple' + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py", + classname = "JournalNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + + # Testing that the exception throw by cached_executor is caught + cached_kinit_executor_mock.reset_mock() + cached_kinit_executor_mock.side_effect = Exception("Invalid command") + + try: + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py", + classname = "JournalNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + except: + self.assertTrue(True) + + # Testing with a security_params which doesn't contains hdfs-site + empty_security_params = {} + empty_security_params['core-site'] = {} + empty_security_params['core-site']['hadoop.security.authentication'] = 'kerberos' + cached_kinit_executor_mock.reset_mock() + get_params_mock.reset_mock() + put_structured_out_mock.reset_mock() + get_params_mock.return_value = empty_security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py", + classname = "JournalNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."}) + + # Testing with not empty result_issues + result_issues_with_params = {} + result_issues_with_params['hdfs-site']="Something bad happened" + + validate_security_config_mock.reset_mock() + get_params_mock.reset_mock() + validate_security_config_mock.return_value = result_issues_with_params + get_params_mock.return_value = security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py", + classname = "JournalNode", + command = "security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file
