Repository: ambari Updated Branches: refs/heads/trunk 4857a6e48 -> 9dd542be2
AMBARI-20779 Create Ranger KMS HDFS audit folder as part of install (mugdha) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9dd542be Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9dd542be Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9dd542be Branch: refs/heads/trunk Commit: 9dd542be2c7bb3b12ea50c8902f3d8e1413eebf3 Parents: 4857a6e Author: Mugdha Varadkar <[email protected]> Authored: Fri Apr 28 13:35:27 2017 +0530 Committer: Gautam Borad <[email protected]> Committed: Sun Apr 30 23:23:08 2017 +0530 ---------------------------------------------------------------------- .../RANGER/0.5.0/role_command_order.json | 2 +- .../RANGER_KMS/0.5.0.2.3/metainfo.xml | 9 ++ .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 20 ++++ .../0.5.0.2.3/package/scripts/params.py | 34 ++++++- .../0.5.0.2.3/role_command_order.json | 2 +- .../stacks/HDP/2.3/role_command_order.json | 4 +- .../stacks/2.5/RANGER_KMS/test_kms_server.py | 98 ++++++++++++++++++++ .../stacks/2.5/configs/ranger-kms-secured.json | 2 +- 8 files changed, 165 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json index df62dfd..073a077 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json +++ b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/role_command_order.json @@ -1,6 +1,6 @@ { "general_deps" : { "_comment" : "dependencies for RANGER", - "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"] + "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml index b20201c..bc0aa74 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml @@ -45,6 +45,15 @@ <primary>true</primary> </log> </logs> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> </component> </components> http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py index 423cdec..5a25b92 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py @@ -506,6 +506,26 @@ def enable_kms_plugin(): mode = 0640 ) + # create ranger kms audit directory + if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/kms", + type="directory", + action="create_on_execute", + owner=params.kms_user, + group=params.kms_group, + mode=0750, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1: Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml') XmlConfig("hdfs-site.xml", http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py index 9fe0a61..2445f2e 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py @@ -29,6 +29,9 @@ from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.get_bare_principal import get_bare_principal from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import get_kinit_path config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -296,4 +299,33 @@ xa_audit_hdfs_is_enabled = default("/configurations/ranger-kms-audit/xasecure.au namenode_host = default("/clusterHostInfo/namenode_host", []) # need this to capture cluster name from where ranger kms plugin is enabled -cluster_name = config['clusterName'] \ No newline at end of file +cluster_name = config['clusterName'] + +has_namenode = len(namenode_host) > 0 + +hdfs_user = default("/configurations/hadoop-env/hdfs_user", None) +hdfs_user_keytab = default("/configurations/hadoop-env/hdfs_user_keytab", None) +hdfs_principal_name = default("/configurations/hadoop-env/hdfs_principal_name", None) +default_fs = default("/configurations/core-site/fs.defaultFS", None) +hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +import functools +# create partial functions with common arguments for every HdfsResource call +# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs +) + +local_component_list = default("/localComponents", []) +has_hdfs_client_on_node = 'HDFS_CLIENT' in local_component_list \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json index 006d177..7ddab41 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/role_command_order.json @@ -1,7 +1,7 @@ { "general_deps" : { "_comment" : "dependencies for RANGER-KMS", - "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"], + "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"], "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json index 4f279c6..5b1b50e 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json @@ -4,8 +4,8 @@ "general_deps" : { "_comment" : "dependencies for all cases", "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], - "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"], - "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"], + "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"], + "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"], "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"], "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"], "SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"], http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py index 7082a33..3d03c5f 100644 --- a/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py +++ b/ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py @@ -151,6 +151,55 @@ class TestRangerKMS(RMFTestCase): mode = 0640 ) + self.assertResourceCalled('HdfsResource', '/ranger/audit', + type = 'directory', + action = ['create_on_execute'], + owner = 'hdfs', + group = 'hdfs', + mode = 0755, + recursive_chmod = True, + user = 'hdfs', + security_enabled = False, + keytab = None, + kinit_path_local = '/usr/bin/kinit', + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + principal_name = None, + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + default_fs = 'hdfs://c6401.ambari.apache.org:8020' + ) + + self.assertResourceCalled('HdfsResource', '/ranger/audit/kms', + type = 'directory', + action = ['create_on_execute'], + owner = 'kms', + group = 'kms', + mode = 0750, + recursive_chmod = True, + user = 'hdfs', + security_enabled = False, + keytab = None, + kinit_path_local = '/usr/bin/kinit', + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + principal_name = None, + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + default_fs = 'hdfs://c6401.ambari.apache.org:8020' + ) + + self.assertResourceCalled('HdfsResource', None, + action = ['execute'], + user = 'hdfs', + security_enabled = False, + keytab = None, + kinit_path_local = '/usr/bin/kinit', + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + principal_name = None, + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + default_fs = 'hdfs://c6401.ambari.apache.org:8020' + ) + self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml', action = ['delete'], ) @@ -537,6 +586,55 @@ class TestRangerKMS(RMFTestCase): mode = 0640 ) + self.assertResourceCalled('HdfsResource', '/ranger/audit', + type = 'directory', + action = ['create_on_execute'], + owner = 'hdfs', + group = 'hdfs', + mode = 0755, + recursive_chmod = True, + user = 'hdfs', + security_enabled = True, + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + kinit_path_local = '/usr/bin/kinit', + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + principal_name = '[email protected]', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + default_fs = 'hdfs://c6401.ambari.apache.org:8020' + ) + + self.assertResourceCalled('HdfsResource', '/ranger/audit/kms', + type = 'directory', + action = ['create_on_execute'], + owner = 'kms', + group = 'kms', + mode = 0750, + recursive_chmod = True, + user = 'hdfs', + security_enabled = True, + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + kinit_path_local = '/usr/bin/kinit', + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + principal_name = '[email protected]', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + default_fs = 'hdfs://c6401.ambari.apache.org:8020' + ) + + self.assertResourceCalled('HdfsResource', None, + action = ['execute'], + user = 'hdfs', + security_enabled = True, + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + kinit_path_local = '/usr/bin/kinit', + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + principal_name = '[email protected]', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + default_fs = 'hdfs://c6401.ambari.apache.org:8020' + ) + self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml', action = ['delete'], ) http://git-wip-us.apache.org/repos/asf/ambari/blob/9dd542be/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json index 7054e8f..4e7d857 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json @@ -751,7 +751,7 @@ "hdfs_log_dir_prefix": "/var/log/hadoop", "hdfs_user_nofile_limit": "128000", "hdfs_user": "hdfs", - "hdfs_principal_name": "[email protected]", + "hdfs_principal_name": "[email protected]", "keyserver_host": " ", "namenode_opt_maxnewsize": "128m", "hdfs_user_keytab": "/etc/security/keytabs/hdfs.headless.keytab",
