Repository: ambari Updated Branches: refs/heads/branch-2.5 7ee057287 -> e2c92d5f6
AMBARI-20047 Repository config user needs to set customized service user in Ranger when service plugin is enabled (Vishal Suvagia via mugdha) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e2c92d5f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e2c92d5f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e2c92d5f Branch: refs/heads/branch-2.5 Commit: e2c92d5f6b5d0ba8c7d749266fa7b44235f7810f Parents: 7ee0572 Author: Vishal Suvagia <[email protected]> Authored: Wed Feb 22 20:22:34 2017 +0530 Committer: Mugdha Varadkar <[email protected]> Committed: Fri Feb 24 10:26:18 2017 +0530 ---------------------------------------------------------------------- .../functions/setup_ranger_plugin_xml.py | 1 + .../ranger-atlas-plugin-properties.xml | 20 + .../ranger-hbase-plugin-properties.xml | 17 + .../ranger-hdfs-plugin-properties.xml | 18 + .../ranger-hive-plugin-properties.xml | 18 + .../ranger-kafka-plugin-properties.xml | 17 + .../ranger-yarn-plugin-properties.xml | 17 + .../stacks/HDP/2.6/services/stack_advisor.py | 119 +++- .../stacks/2.6/common/test_stack_advisor.py | 539 ++++++++++++++++++- 9 files changed, 759 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 56c46dd..04a5bb1 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -88,6 +88,7 @@ def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, if 'serviceName' in json_data and json_data['serviceName'] == repo_name: service_name_exist = True Logger.info("Skipping Ranger API calls, as policy cache file exists for {0}".format(service_name)) + Logger.warning("If service name for {0} is not created on Ranger Admin UI, then to re-create it delete policy cache file: {1}".format(service_name, policycache_json_file)) break except Exception, err: Logger.error("Error occurred while fetching service name from policy cache file.\nError: {0}".format(err)) http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml index d8885e5..c53f203 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/ranger-atlas-plugin-properties.xml @@ -67,5 +67,25 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>REPOSITORY_CONFIG_USERNAME</name> + <value>admin</value> + <display-name>Ranger repository config user</display-name> + <description>Used for repository creation on ranger admin + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>REPOSITORY_CONFIG_PASSWORD</name> + <value>admin</value> + <display-name>Ranger repository config password</display-name> + <property-type>PASSWORD</property-type> + <description>Used for repository creation on ranger admin + </description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/configuration/ranger-hbase-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/configuration/ranger-hbase-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/configuration/ranger-hbase-plugin-properties.xml index d8885e5..d19aa1c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/configuration/ranger-hbase-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/configuration/ranger-hbase-plugin-properties.xml @@ -67,5 +67,22 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>REPOSITORY_CONFIG_USERNAME</name> + <value>hbase</value> + <display-name>Ranger repository config user</display-name> + <description>Used for repository creation on ranger admin</description> + <depends-on> + <property> + <type>ranger-hbase-plugin-properties</type> + <name>ranger-hbase-plugin-enabled</name> + </property> + <property> + <type>hbase-env</type> + <name>hbase_user</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml index 6e93879..d8639fc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml @@ -67,4 +67,22 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>REPOSITORY_CONFIG_USERNAME</name> + <value>hadoop</value> + <display-name>Ranger repository config user</display-name> + <description>Used for repository creation on ranger admin + </description> + <depends-on> + <property> + <type>ranger-hdfs-plugin-properties</type> + <name>ranger-hdfs-plugin-enabled</name> + </property> + <property> + <type>hadoop-env</type> + <name>hdfs_user</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/ranger-hive-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/ranger-hive-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/ranger-hive-plugin-properties.xml index d8885e5..cc250bb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/ranger-hive-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/ranger-hive-plugin-properties.xml @@ -67,5 +67,23 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>REPOSITORY_CONFIG_USERNAME</name> + <value>hive</value> + <display-name>Ranger repository config user</display-name> + <description>Used for repository creation on ranger admin + </description> + <depends-on> + <property> + <type>hive-env</type> + <name>hive_security_authorization</name> + </property> + <property> + <type>hive-env</type> + <name>hive_user</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml index d8885e5..8b72e7d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml @@ -67,5 +67,22 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>REPOSITORY_CONFIG_USERNAME</name> + <value>kafka</value> + <display-name>Ranger repository config user</display-name> + <description>Used for repository creation on ranger admin</description> + <depends-on> + <property> + <type>ranger-kafka-plugin-properties</type> + <name>ranger-kafka-plugin-enabled</name> + </property> + <property> + <type>kafka-env</type> + <name>kafka_user</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/ranger-yarn-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/ranger-yarn-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/ranger-yarn-plugin-properties.xml index d8885e5..c2e5c75 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/ranger-yarn-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/ranger-yarn-plugin-properties.xml @@ -67,5 +67,22 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>REPOSITORY_CONFIG_USERNAME</name> + <value>yarn</value> + <display-name>Ranger repository config user</display-name> + <description>Used for repository creation on ranger admin</description> + <depends-on> + <property> + <type>ranger-yarn-plugin-properties</type> + <name>ranger-yarn-plugin-enabled</name> + </property> + <property> + <type>yarn-env</type> + <name>yarn_user</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 3dd1bdd..0968d8b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -30,11 +30,16 @@ class HDP26StackAdvisor(HDP25StackAdvisor): def getServiceConfigurationRecommenderDict(self): parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { - "DRUID": self.recommendDruidConfigurations, - "ATLAS": self.recommendAtlasConfigurations, - "TEZ": self.recommendTezConfigurations, - "RANGER": self.recommendRangerConfigurations, - "RANGER_KMS": self.recommendRangerKMSConfigurations + "DRUID": self.recommendDruidConfigurations, + "ATLAS": self.recommendAtlasConfigurations, + "TEZ": self.recommendTezConfigurations, + "RANGER": self.recommendRangerConfigurations, + "RANGER_KMS": self.recommendRangerKMSConfigurations, + "HDFS": self.recommendHDFSConfigurations, + "HIVE": self.recommendHIVEConfigurations, + "HBASE": self.recommendHBASEConfigurations, + "YARN": self.recommendYARNConfigurations, + "KAFKA": self.recommendKAFKAConfigurations } parentRecommendConfDict.update(childRecommendConfDict) return parentRecommendConfDict @@ -151,6 +156,24 @@ class HDP26StackAdvisor(HDP25StackAdvisor): else: putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false") + if 'yarn-env' in services['configurations'] and 'yarn_user' in services['configurations']['yarn-env']['properties']: + yarn_user = services['configurations']['yarn-env']['properties']['yarn_user'] + else: + yarn_user = 'yarn' + if 'ranger-yarn-plugin-properties' in configurations and 'ranger-yarn-plugin-enabled' in configurations['ranger-yarn-plugin-properties']['properties']: + ranger_yarn_plugin_enabled = (configurations['ranger-yarn-plugin-properties']['properties']['ranger-yarn-plugin-enabled'].lower() == 'Yes'.lower()) + elif 'ranger-yarn-plugin-properties' in services['configurations'] and 'ranger-yarn-plugin-enabled' in services['configurations']['ranger-yarn-plugin-properties']['properties']: + ranger_yarn_plugin_enabled = (services['configurations']['ranger-yarn-plugin-properties']['properties']['ranger-yarn-plugin-enabled'].lower() == 'Yes'.lower()) + else: + ranger_yarn_plugin_enabled = False + + if ranger_yarn_plugin_enabled and 'ranger-yarn-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-yarn-plugin-properties']['properties']: + Logger.info("Setting Yarn Repo user for Ranger.") + putRangerYarnPluginProperty = self.putProperty(configurations, "ranger-yarn-plugin-properties", services) + putRangerYarnPluginProperty("REPOSITORY_CONFIG_USERNAME",yarn_user) + else: + Logger.info("Not setting Yarn Repo user for Ranger.") + def getMetadataConnectionString(self, database_type): driverDict = { 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true', @@ -320,4 +343,88 @@ class HDP26StackAdvisor(HDP25StackAdvisor): if ranger_kms_ssl_enabled: putRangerKmsEnvProperty("kms_port", ranger_kms_ssl_port) else: - putRangerKmsEnvProperty("kms_port", "9292") \ No newline at end of file + putRangerKmsEnvProperty("kms_port", "9292") + + def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts): + super(HDP26StackAdvisor, self).recommendHDFSConfigurations(configurations, clusterData, services, hosts) + if 'hadoop-env' in services['configurations'] and 'hdfs_user' in services['configurations']['hadoop-env']['properties']: + hdfs_user = services['configurations']['hadoop-env']['properties']['hdfs_user'] + else: + hdfs_user = 'hadoop' + + if 'ranger-hdfs-plugin-properties' in configurations and 'ranger-hdfs-plugin-enabled' in configurations['ranger-hdfs-plugin-properties']['properties']: + ranger_hdfs_plugin_enabled = (configurations['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled'].lower() == 'Yes'.lower()) + elif 'ranger-hdfs-plugin-properties' in services['configurations'] and 'ranger-hdfs-plugin-enabled' in services['configurations']['ranger-hdfs-plugin-properties']['properties']: + ranger_hdfs_plugin_enabled = (services['configurations']['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled'].lower() == 'Yes'.lower()) + else: + ranger_hdfs_plugin_enabled = False + + if ranger_hdfs_plugin_enabled and 'ranger-hdfs-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hdfs-plugin-properties']['properties']: + Logger.info("Setting HDFS Repo user for Ranger.") + putRangerHDFSPluginProperty = self.putProperty(configurations, "ranger-hdfs-plugin-properties", services) + putRangerHDFSPluginProperty("REPOSITORY_CONFIG_USERNAME",hdfs_user) + else: + Logger.info("Not setting HDFS Repo user for Ranger.") + + def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts): + super(HDP26StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts) + if 'hive-env' in services['configurations'] and 'hive_user' in services['configurations']['hive-env']['properties']: + hive_user = services['configurations']['hive-env']['properties']['hive_user'] + else: + hive_user = 'hive' + + if 'hive-env' in configurations and 'hive_security_authorization' in configurations['hive-env']['properties']: + ranger_hive_plugin_enabled = (configurations['hive-env']['properties']['hive_security_authorization'].lower() == 'ranger') + elif 'hive-env' in services['configurations'] and 'hive_security_authorization' in services['configurations']['hive-env']['properties']: + ranger_hive_plugin_enabled = (services['configurations']['hive-env']['properties']['hive_security_authorization'].lower() == 'ranger') + else : + ranger_hive_plugin_enabled = False + + if ranger_hive_plugin_enabled and 'ranger-hive-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hive-plugin-properties']['properties']: + Logger.info("Setting Hive Repo user for Ranger.") + putRangerHivePluginProperty = self.putProperty(configurations, "ranger-hive-plugin-properties", services) + putRangerHivePluginProperty("REPOSITORY_CONFIG_USERNAME",hive_user) + else: + Logger.info("Not setting Hive Repo user for Ranger.") + + def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): + super(HDP26StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts) + if 'hbase-env' in services['configurations'] and 'hbase_user' in services['configurations']['hbase-env']['properties']: + hbase_user = services['configurations']['hbase-env']['properties']['hbase_user'] + else: + hbase_user = 'hbase' + + if 'ranger-hbase-plugin-properties' in configurations and 'ranger-hbase-plugin-enabled' in configurations['ranger-hbase-plugin-properties']['properties']: + ranger_hbase_plugin_enabled = (configurations['ranger-hbase-plugin-properties']['properties']['ranger-hbase-plugin-enabled'].lower() == 'Yes'.lower()) + elif 'ranger-hbase-plugin-properties' in services['configurations'] and 'ranger-hbase-plugin-enabled' in services['configurations']['ranger-hbase-plugin-properties']['properties']: + ranger_hbase_plugin_enabled = (services['configurations']['ranger-hbase-plugin-properties']['properties']['ranger-hbase-plugin-enabled'].lower() == 'Yes'.lower()) + else: + ranger_hbase_plugin_enabled = False + + if ranger_hbase_plugin_enabled and 'ranger-hbase-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hbase-plugin-properties']['properties']: + Logger.info("Setting Hbase Repo user for Ranger.") + putRangerHbasePluginProperty = self.putProperty(configurations, "ranger-hbase-plugin-properties", services) + putRangerHbasePluginProperty("REPOSITORY_CONFIG_USERNAME",hbase_user) + else: + Logger.info("Not setting Hbase Repo user for Ranger.") + + def recommendKAFKAConfigurations(self, configurations, clusterData, services, hosts): + super(HDP26StackAdvisor, self).recommendKAFKAConfigurations(configurations, clusterData, services, hosts) + if 'kafka-env' in services['configurations'] and 'kafka_user' in services['configurations']['kafka-env']['properties']: + kafka_user = services['configurations']['kafka-env']['properties']['kafka_user'] + else: + kafka_user = "kafka" + + if 'ranger-kafka-plugin-properties' in configurations and 'ranger-kafka-plugin-enabled' in configurations['ranger-kafka-plugin-properties']['properties']: + ranger_kafka_plugin_enabled = (configurations['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled'].lower() == 'Yes'.lower()) + elif 'ranger-kafka-plugin-properties' in services['configurations'] and 'ranger-kafka-plugin-enabled' in services['configurations']['ranger-kafka-plugin-properties']['properties']: + ranger_kafka_plugin_enabled = (services['configurations']['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled'].lower() == 'Yes'.lower()) + else: + ranger_kafka_plugin_enabled = False + + if ranger_kafka_plugin_enabled and 'ranger-kafka-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-kafka-plugin-properties']['properties']: + Logger.info("Setting Kafka Repo user for Ranger.") + putRangerKafkaPluginProperty = self.putProperty(configurations, "ranger-kafka-plugin-properties", services) + putRangerKafkaPluginProperty("REPOSITORY_CONFIG_USERNAME",kafka_user) + else: + Logger.info("Not setting Kafka Repo user for Ranger.") http://git-wip-us.apache.org/repos/asf/ambari/blob/e2c92d5f/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index c15eaf1..5bfa1a9 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -20,7 +20,7 @@ import json import os from unittest import TestCase from mock.mock import patch - +import socket class TestHDP26StackAdvisor(TestCase): def setUp(self): @@ -849,6 +849,543 @@ class TestHDP26StackAdvisor(TestCase): self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None) self.assertEquals(recommendedConfigurations, expected) + def test_recommendHDFSConfigurations(self): + ambariHostName = socket.getfqdn() + configurations = { + "ranger-hdfs-plugin-properties": { + "properties": { + "ranger-hdfs-plugin-enabled": "Yes", + "REPOSITORY_CONFIG_USERNAME":"hadoop" + } + }, + "hadoop-env":{ + "properties":{ + "hdfs_user":"custom_hdfs" + } + } + } + clusterData = { + "totalAvailableRam": 2048, + "hBaseInstalled": True, + "hbaseRam": 112, + "reservedRam": 128 + } + + hosts = { + "items": [ + { + "Hosts": { + "disk_info": [{ + "size": '8', + "mountpoint": "/" + }] + } + }]} + + services = { + "services": + [ + { + "StackServices": { + "service_name" : "HDFS", + "service_version" : "2.7.0.2.6" + }, + "components": [ + ] + } + ], + "Versions": { + "stack_version": "2.6" + }, + "configurations": configurations, + "ambari-server-properties": {"ambari-server.user":"ambari_user"} + } + + + expected = { + 'core-site': { + 'properties': { + 'hadoop.proxyuser.ambari_user.groups': '*', + 'hadoop.proxyuser.custom_hdfs.groups': '*', + 'hadoop.proxyuser.custom_hdfs.hosts': '*', + 'hadoop.proxyuser.ambari_user.hosts': ambariHostName + }, + 'property_attributes': { + 'hadoop.security.key.provider.path': { + 'delete': 'true' + } + } + }, + 'hadoop-env': { + 'properties': { + 'hdfs_user': 'custom_hdfs', + 'namenode_heapsize': '1024', + 'namenode_opt_maxnewsize': '128', + 'namenode_opt_newsize': '128' + } + }, + 'hdfs-site': { + 'properties': { + 'dfs.datanode.data.dir': '/hadoop/hdfs/data', + 'dfs.datanode.failed.volumes.tolerated': '0', + 'dfs.datanode.max.transfer.threads': '16384', + 'dfs.namenode.name.dir': '/hadoop/hdfs/namenode', + 'dfs.namenode.handler.count': '100', + 'dfs.namenode.checkpoint.dir': '/hadoop/hdfs/namesecondary', + 'dfs.namenode.safemode.threshold-pct': '1.000', + 'dfs.namenode.inode.attributes.provider.class': 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer', + 'dfs.datanode.du.reserved': '1073741824' + }, + 'property_attributes': { + 'dfs.datanode.failed.volumes.tolerated': { + 'maximum': '1' + }, + 'dfs.encryption.key.provider.uri': { + 'delete': 'true' + } + } + }, + 'ranger-hdfs-plugin-properties': { + 'properties': { + 'ranger-hdfs-plugin-enabled': 'Yes', + 'REPOSITORY_CONFIG_USERNAME': 'custom_hdfs' + } + } + } + + self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations,expected) + configurations['hadoop-env']['properties']['hdfs_user'] = 'hadoop' + expected['hadoop-env']['properties']['hdfs_user'] = 'hadoop' + expected['ranger-hdfs-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'hadoop' + expected['core-site']['properties']['hadoop.proxyuser.hadoop.hosts'] = '*' + expected['core-site']['properties']['hadoop.proxyuser.hadoop.groups'] = '*' + self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations,expected) + + + def test_recommendHiveConfigurations(self): + configurations = { + "ranger-hive-plugin-properties": { + "properties": { + "ranger-hive-plugin-enabled": "Yes", + "REPOSITORY_CONFIG_USERNAME":"hive" + } + }, + "hive-env":{ + "properties":{ + "hive_security_authorization":"ranger", + "hive_user":"custom_hive" + } + } + } + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256, + "yarnMinContainerSize": 256 + } + + hosts = {} + + services = { + "services": + [ + { + "StackServices": { + "service_name" : "HIVE", + "service_version" : "1.2.1.2.6" + }, + "components": [ + ] + } + ], + "Versions": { + "stack_name" : "HDP", + "stack_version": "2.6" + }, + "configurations": configurations, + "ambari-server-properties": {"ambari-server.user":"ambari_user"} + } + + + expected = { + 'yarn-env': { + 'properties': { + 'min_user_id': '500', + 'service_check.queue.name': 'default' + } + }, + 'ranger-hive-plugin-properties': { + 'properties': { + 'ranger-hive-plugin-enabled': 'Yes', + 'REPOSITORY_CONFIG_USERNAME': 'custom_hive' + } + }, + 'webhcat-site': { + 'properties': { + 'templeton.hadoop.queue.name': 'default' + } + }, + 'hive-interactive-env': { + 'properties': { + 'enable_hive_interactive': 'false' + }, + 'property_attributes': { + 'num_llap_nodes': { + 'read_only': 'true' + } + } + }, + 'hive-env': { + 'properties': { + 'hive.atlas.hook': 'false', + 'hive_security_authorization': 'ranger', + 'hive_exec_orc_storage_strategy': 'SPEED', + 'hive_timeline_logging_enabled': 'true', + 'hive_txn_acid': 'off', + 'hive_user': 'custom_hive' + } + }, + 'hiveserver2-site': { + 'properties': { + 'hive.security.authorization.enabled': 'true', + 'hive.conf.restricted.list': 'hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled', + 'hive.security.authenticator.manager': 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator', + 'hive.security.authorization.manager': 'org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory' + } + }, + 'hive-site': { + 'properties': { + 'hive.tez.container.size': '768', + 'hive.exec.orc.default.stripe.size': '67108864', + 'hive.execution.engine': 'mr', + 'hive.vectorized.execution.reduce.enabled': 'false', + 'hive.compactor.worker.threads': '0', + 'hive.compactor.initiator.on': 'false', + 'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', + 'hive.compute.query.using.stats': 'true', + 'hive.exec.orc.default.compress': 'ZLIB', + 'hive.exec.orc.encoding.strategy': 'SPEED', + 'hive.server2.tez.initialize.default.sessions': 'false', + 'hive.security.authorization.enabled': 'true', + 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', + 'hive.server2.tez.default.queues': 'default', + 'hive.prewarm.enabled': 'false', + 'hive.exec.orc.compression.strategy': 'SPEED', + 'hive.optimize.index.filter': 'true', + 'hive.auto.convert.join.noconditionaltask.size': '214748364', + 'hive.vectorized.execution.enabled': 'true', + 'hive.exec.reducers.bytes.per.reducer': '67108864', + 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager', + 'hive.server2.tez.sessions.per.default.queue': '1', + 'hive.prewarm.numcontainers': '3', + 'hive.tez.dynamic.partition.pruning': 'true', + 'hive.tez.auto.reducer.parallelism': 'true', + 'hive.server2.use.SSL': 'false', + 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', + 'hive.support.concurrency': 'false', + 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps', + 'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider', + 'hive.exec.dynamic.partition.mode': 'strict', + 'hive.optimize.sort.dynamic.partition': 'false', + 'hive.server2.enable.doAs': 'false' + }, + 'property_attributes': { + 'hive.tez.container.size': { + 'minimum': '256', + 'maximum': '768' + }, + 'atlas.cluster.name': { + 'delete': 'true' + }, + 'hive.server2.tez.default.queues': { + 'entries': [ + { + 'value': 'default', + 'label': 'default queue' + } + ] + }, + 'datanucleus.rdbms.datastoreAdapterClassName': { + 'delete': 'true' + }, + 'hive.auto.convert.join.noconditionaltask.size': { + 'maximum': '644245094' + }, + 'atlas.rest.address': { + 'delete': 'true' + } + } + }, + 'hive-interactive-site': { + 'properties': {} + }, + 'yarn-site': { + 'properties': { + 'hadoop.registry.rm.enabled': 'false', + 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes': '', + 'yarn.scheduler.minimum-allocation-vcores': '1', + 'yarn.scheduler.maximum-allocation-vcores': '4', + 'yarn.nodemanager.resource.memory-mb': '768', + 'yarn.scheduler.minimum-allocation-mb': '256', + 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', + 'yarn.nodemanager.resource.cpu-vcores': '4', + 'yarn.scheduler.maximum-allocation-mb': '768', + 'yarn.nodemanager.linux-container-executor.group': 'hadoop' + }, + 'property_attributes': { + 'yarn.authorization-provider': { + 'delete': 'true' + } + } + } + } + + self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations,expected) + configurations['hive-env']['properties']['hive_user'] = 'hive' + expected['hive-env']['properties']['hive_user'] = 'hive' + expected['ranger-hive-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'hive' + self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations,expected) + + + def test_recommendHBASEConfigurations(self): + configurations = { + "ranger-hbase-plugin-properties": { + "properties": { + "ranger-hbase-plugin-enabled": "Yes", + "REPOSITORY_CONFIG_USERNAME":"hbase" + } + }, + "hbase-env":{ + "properties":{ + "hbase_user":"custom_hbase" + } + } + } + + services = { + "services": [{ + "StackServices": { + "service_name" : "HBASE", + "service_version" : "1.1.2.2.6" + }, + "components": [ + ] + } + ], + "Versions": { + "stack_name" : "HDP", + "stack_version": "2.6" + }, + "configurations": configurations, + "ambari-server-properties": {"ambari-server.user":"ambari_user"} + } + + + clusterData = { + "totalAvailableRam": 2048, + "hBaseInstalled": True, + "hbaseRam": 112, + "reservedRam": 128 + } + expected = { + 'hbase-site': { + 'properties': { + 'hbase.regionserver.wal.codec': 'org.apache.hadoop.hbase.regionserver.wal.WALCellCodec', + 'hbase.master.ui.readonly': 'false', + 'hbase.security.authorization': 'true', + 'hbase.bucketcache.percentage.in.combinedcache': '1.0000', + 'hbase.regionserver.global.memstore.size': '0.4', + 'hfile.block.cache.size': '0.4', + 'hbase.coprocessor.region.classes': 'org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor', + 'hbase.bucketcache.size': '92160', + 'hbase.coprocessor.regionserver.classes': 'org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor', + 'hbase.coprocessor.master.classes': 'org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor', + 'hbase.bucketcache.ioengine': 'offheap' + }, + 'property_attributes': { + 'hbase.bucketcache.percentage.in.combinedcache': { + 'delete': 'true' + }, + 'hbase.region.server.rpc.scheduler.factory.class': { + 'delete': 'true' + } + } + }, + 'ranger-hbase-plugin-properties': { + 'properties': { + 'REPOSITORY_CONFIG_USERNAME': 'custom_hbase', + 'ranger-hbase-plugin-enabled': 'Yes' + } + }, + 'hbase-env': { + 'properties': { + 'hbase_user': 'custom_hbase', + 'hbase_master_heapsize': '1024', + 'hbase_regionserver_heapsize': '20480', + 'hbase_max_direct_memory_size': '94208' + } + }, + 'core-site': { + 'properties': {} + } + } + self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, expected) + configurations['hbase-env']['properties']['hbase_user'] = 'hbase' + expected['hbase-env']['properties']['hbase_user'] = 'hbase' + expected['ranger-hbase-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'hbase' + self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, expected) + + def test_recommendYARNConfigurations(self): + configurations = { + "yarn-env": { + "properties": { + "yarn_user" : "custom_yarn" + } + }, + "ranger-yarn-plugin-properties": { + "properties": { + "ranger-yarn-plugin-enabled" : "Yes", + "REPOSITORY_CONFIG_USERNAME":"yarn" + } + } + } + services = { + "services" : [{ + "StackServices": { + "service_name" : "YARN", + "service_version" : "2.7.3.2.6" + }, + "components": [] + } + ], + "configurations": configurations + } + + + clusterData = { + "cpu": 4, + "containers" : 5, + "ramPerContainer": 256, + "yarnMinContainerSize": 256 + } + expected = { + 'yarn-env': { + 'properties': { + 'yarn_user': 'custom_yarn', + 'service_check.queue.name': 'default', + 'min_user_id': '500' + } + }, + 'ranger-yarn-plugin-properties': { + 'properties': { + 'ranger-yarn-plugin-enabled': 'Yes', + 'REPOSITORY_CONFIG_USERNAME': 'custom_yarn' + } + }, + 'yarn-site': { + 'properties': { + 'hadoop.registry.rm.enabled': 'false', + 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes': '', + 'yarn.authorization-provider': 'org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer', + 'yarn.acl.enable': 'true', + 'yarn.scheduler.minimum-allocation-vcores': '1', + 'yarn.scheduler.maximum-allocation-vcores': '4', + 'yarn.nodemanager.resource.memory-mb': '1280', + 'yarn.scheduler.minimum-allocation-mb': '256', + 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', + 'yarn.nodemanager.resource.cpu-vcores': '4', + 'yarn.scheduler.maximum-allocation-mb': '1280', + 'yarn.nodemanager.linux-container-executor.group': 'hadoop' + } + } + } + + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, expected) + configurations['yarn-env']['properties']['yarn_user'] = 'yarn' + expected['yarn-env']['properties']['yarn_user'] = 'yarn' + expected['ranger-yarn-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'yarn' + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, expected) + + def test_recommendKAFKAConfigurations(self): + configurations = { + "kafka-env": { + "properties": { + "kafka_user" : "custom_kafka" + } + }, + "ranger-kafka-plugin-properties": { + "properties": { + "ranger-kafka-plugin-enabled" : "Yes", + "REPOSITORY_CONFIG_USERNAME":"kafka" + } + } + } + clusterData = [] + services = { + "services" : [{ + "StackServices": { + "service_name" : "KAFKA", + "service_version" : "0.10.0.2.6" + }, + "components": [] + } + ], + "configurations": configurations + } + + expected = { + 'kafka-env': { + 'properties': { + 'kafka_user': 'custom_kafka' + } + }, + 'kafka-log4j': { + 'properties': {} + }, + 'kafka-broker': { + 'properties': {}, + 'property_attributes': { + 'principal.to.local.class': { + 'delete': 'true' + }, + 'super.users': { + 'delete': 'true' + }, + 'security.inter.broker.protocol': { + 'delete': 'true' + }, + 'authorizer.class.name': { + 'delete': 'true' + } + } + }, + 'ranger-kafka-plugin-properties': { + 'properties': { + 'ranger-kafka-plugin-enabled': 'Yes', + 'REPOSITORY_CONFIG_USERNAME': 'custom_kafka' + } + } + } + + self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, expected) + configurations['kafka-env']['properties']['kafka_user'] = 'kafka' + expected['kafka-env']['properties']['kafka_user'] = 'kafka' + expected['ranger-kafka-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'kafka' + self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, expected) + def load_json(self, filename): file = os.path.join(self.testDirectory, filename) with open(file, 'rb') as f:
