Repository: ambari Updated Branches: refs/heads/branch-2.4 be18a92c9 -> 0ad16dfd2
AMBARI-17415: Ambari configuration for ranger-tagsync needs to support property for atlas keystore filename (Mugdha Varadkar via jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0ad16dfd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0ad16dfd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0ad16dfd Branch: refs/heads/branch-2.4 Commit: 0ad16dfd26ceefcdeae672a660073391768cfaf6 Parents: be18a92 Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Thu Jul 7 00:32:04 2016 -0700 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Thu Jul 7 00:33:31 2016 -0700 ---------------------------------------------------------------------- .../0.6.0/configuration/ranger-tagsync-site.xml | 31 ++++++++++++++++++- .../stacks/HDP/2.2/services/stack_advisor.py | 31 +++++++++++-------- .../stacks/HDP/2.3/services/stack_advisor.py | 29 ++++++++++++++++-- .../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 7 +++++ .../configuration/ranger-tagsync-site.xml | 9 ++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 21 +++++++++++-- .../stacks/2.2/common/test_stack_advisor.py | 28 +++++++++++++++-- .../stacks/2.3/common/test_stack_advisor.py | 32 ++++++++++++++++++++ 8 files changed, 168 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml index 7985f58..73b8227 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml @@ -88,7 +88,7 @@ <property> <name>ranger.tagsync.source.atlasrest.download.interval.millis</name> <display-name>AtlasREST Source: Atlas source download interval</display-name> - <value/> + <value>60000</value> <value-attributes> <empty-value-valid>true</empty-value-valid> </value-attributes> @@ -137,6 +137,20 @@ <empty-value-valid>true</empty-value-valid> </value-attributes> <on-ambari-upgrade add="true"/> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.server.http.port</name> + </property> + <property> + <type>application-properties</type> + <name>atlas.server.https.port</name> + </property> + <property> + <type>application-properties</type> + <name>atlas.enableTLS</name> + </property> + </depends-on> </property> <property> <name>ranger.tagsync.kerberos.principal</name> @@ -162,4 +176,19 @@ <description/> <on-ambari-upgrade add="true"/> </property> + <property> + <name>ranger.tagsync.source.atlasrest.keystore.filename</name> + <value>/etc/ranger/tagsync/conf/atlasuser.jceks</value> + <description>Tagsync atlasrest keystore file</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>ranger.tagsync.source.atlasrest.username</name> + <value>admin</value> + <description/> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 38586e4..e570a5b7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -229,7 +229,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): keyserverHostsString = services["configurations"]["hadoop-env"]["properties"]["keyserver_host"] keyserverPortString = services["configurations"]["hadoop-env"]["properties"]["keyserver_port"] - # Irrespective of what hadoop-env has, if Ranger-KMS is installed, we use its values. + # Irrespective of what hadoop-env has, if Ranger-KMS is installed, we use its values. rangerKMSServerHosts = self.getHostsWithComponent("RANGER_KMS", "RANGER_KMS_SERVER", services, hosts) if rangerKMSServerHosts is not None and len(rangerKMSServerHosts) > 0: rangerKMSServerHostsArray = [] @@ -1134,7 +1134,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): validationItems.append({"config-name" : address_property, "item" : self.getErrorItem(address_property + " does not contain a valid host:port authority: " + value)}) - #Adding Ranger Plugin logic here + #Adding Ranger Plugin logic here ranger_plugin_properties = getSiteProperties(configurations, "ranger-hdfs-plugin-properties") ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties else 'No' servicesList = [service["StackServices"]["service_name"] for service in services["services"]] @@ -1237,8 +1237,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts): hive_server2 = properties - validationItems = [] - #Adding Ranger Plugin logic here + validationItems = [] + #Adding Ranger Plugin logic here ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties") hive_env_properties = getSiteProperties(configurations, "hive-env") ranger_plugin_enabled = 'hive_security_authorization' in hive_env_properties and hive_env_properties['hive_security_authorization'].lower() == 'ranger' @@ -1376,7 +1376,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "item": self.getWarnItem( "{0} and {1} sum should not exceed {2}".format(prop_name1, prop_name2, props_max_sum))}) - #Adding Ranger Plugin logic here + #Adding Ranger Plugin logic here ranger_plugin_properties = getSiteProperties(configurations, "ranger-hbase-plugin-properties") ranger_plugin_enabled = ranger_plugin_properties['ranger-hbase-plugin-enabled'] if ranger_plugin_properties else 'No' prop_name = 'hbase.security.authorization' @@ -1430,7 +1430,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "item": self.getWarnItem( "If bucketcache ioengine is enabled, {0} should be set".format(prop_name3))}) - # Validate hbase.security.authentication. + # Validate hbase.security.authentication. # Kerberos works only when security enabled. if "hbase.security.authentication" in properties: hbase_security_kerberos = properties["hbase.security.authentication"].lower() == "kerberos" @@ -1505,6 +1505,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): validationItems = [] ranger_plugin_properties = getSiteProperties(configurations, "ranger-storm-plugin-properties") ranger_plugin_enabled = ranger_plugin_properties['ranger-storm-plugin-enabled'] if ranger_plugin_properties else 'No' + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] if ranger_plugin_enabled.lower() == 'yes': # ranger-hdfs-plugin must be enabled in ranger-env ranger_env = getServicesSiteProperties(services, 'ranger-env') @@ -1513,6 +1514,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor): validationItems.append({"config-name": 'ranger-storm-plugin-enabled', "item": self.getWarnItem( "ranger-storm-plugin-properties/ranger-storm-plugin-enabled must correspond ranger-env/ranger-storm-plugin-enabled")}) + if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()) and not 'KERBEROS' in servicesList: + validationItems.append({"config-name": "ranger-storm-plugin-enabled", + "item": self.getWarnItem( + "Ranger Storm plugin should not be enabled in non-kerberos environment.")}) + return self.toConfigurationValidationProblems(validationItems, "ranger-storm-plugin-properties") def validateYARNEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): @@ -1546,13 +1552,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "ranger-yarn-plugin-properties") def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts): + ranger_env_properties = properties validationItems = [] - if "ranger-storm-plugin-enabled" in properties and "ranger-storm-plugin-enabled" in recommendedDefaults and \ - properties["ranger-storm-plugin-enabled"] != recommendedDefaults["ranger-storm-plugin-enabled"]: - validationItems.append({"config-name": "ranger-storm-plugin-enabled", - "item": self.getWarnItem( - "Ranger Storm plugin should not be enabled in non-kerberos environment.")}) - + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if "ranger-storm-plugin-enabled" in ranger_env_properties and ranger_env_properties['ranger-storm-plugin-enabled'].lower() == 'yes' and not 'KERBEROS' in servicesList: + validationItems.append({"config-name": "ranger-storm-plugin-enabled", + "item": self.getWarnItem("Ranger Storm plugin should not be enabled in non-kerberos environment.")}) return self.toConfigurationValidationProblems(validationItems, "ranger-env") def getMastersWithMultipleInstances(self): @@ -1573,7 +1578,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def getAffectedConfigs(self, services): affectedConfigs = super(HDP22StackAdvisor, self).getAffectedConfigs(services) - # There are configs that are not defined in the stack but added/removed by + # There are configs that are not defined in the stack but added/removed by # stack-advisor. Here we add such configs in order to clear the config # filtering down in base class configsList = [affectedConfig["type"] + "/" + affectedConfig["name"] for affectedConfig in affectedConfigs] http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 879008b..460aea3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -724,7 +724,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "HBASE": {"hbase-site": self.validateHBASEConfigurations}, "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations}, "YARN": {"yarn-site": self.validateYARNConfigurations}, - "RANGER": {"admin-properties": self.validateRangerAdminConfigurations} + "RANGER": {"admin-properties": self.validateRangerAdminConfigurations, + "ranger-env": self.validateRangerConfigurationsEnv} } self.mergeValidators(parentValidators, childValidators) return parentValidators @@ -907,6 +908,11 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "If Ranger Kafka Plugin is enabled."\ "{0} needs to be set to {1}".format(prop_name,prop_val))}) + if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()) and not 'KERBEROS' in servicesList: + validationItems.append({"config-name": "ranger-kafka-plugin-enabled", + "item": self.getWarnItem( + "Ranger Kafka plugin should not be enabled in non-kerberos environment.")}) + return self.toConfigurationValidationProblems(validationItems, "kafka-broker") def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): @@ -917,7 +923,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled'] if yarn_resource_proxy_enabled.lower() == 'true': validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled', - "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled "\ + "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled " \ "property value as false under yarn-site" )}) @@ -937,3 +943,22 @@ class HDP23StackAdvisor(HDP22StackAdvisor): 'item':self.getWarnItem('Ranger External URL should not contain trailing slash "/"')}) return self.toConfigurationValidationProblems(validationItems,'admin-properties') + def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts): + parentValidationProblems = super(HDP23StackAdvisor, self).validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, hosts) + ranger_env_properties = properties + validationItems = [] + security_enabled = False + + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if 'KERBEROS' in servicesList: + security_enabled = True + + if "ranger-kafka-plugin-enabled" in ranger_env_properties and ranger_env_properties["ranger-kafka-plugin-enabled"].lower() == 'yes' and not security_enabled: + validationItems.append({"config-name": "ranger-kafka-plugin-enabled", + "item": self.getWarnItem( + "Ranger Kafka plugin should not be enabled in non-kerberos environment.")}) + + validationProblems = self.toConfigurationValidationProblems(validationItems, "ranger-env") + validationProblems.extend(parentValidationProblems) + return validationProblems + http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml index 020e339..6e8308a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml @@ -62,6 +62,13 @@ </osSpecific> </osSpecifics> + <configuration-dependencies> + <config-type>ranger-atlas-audit</config-type> + <config-type>ranger-atlas-plugin-properties</config-type> + <config-type>ranger-atlas-policymgr-ssl</config-type> + <config-type>ranger-atlas-security</config-type> + </configuration-dependencies> + <requiredServices> <service>KAFKA</service> </requiredServices> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml index c3fe932..6a0991b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml @@ -34,4 +34,13 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> + <property> + <name>ranger.tagsync.source.atlasrest.keystore.filename</name> + <value>/usr/hdp/current/ranger-tagsync/conf/atlasuser.jceks</value> + <description>Tagsync atlasrest keystore file</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 1d092cd..2ca8c05 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1399,12 +1399,29 @@ class HDP25StackAdvisor(HDP24StackAdvisor): has_ranger_tagsync = len(ranger_tagsync_host) > 0 if 'ATLAS' in servicesList and has_ranger_tagsync: + atlas_hosts = self.getHostNamesWithComponent("ATLAS", "ATLAS_SERVER", services) + atlas_host = 'localhost' if len(atlas_hosts) == 0 else atlas_hosts[0] + protocol = 'http' + atlas_port = '21000' + + if 'application-properties' in services['configurations'] and 'atlas.enableTLS' in services['configurations']['application-properties']['properties'] \ + and services['configurations']['application-properties']['properties']['atlas.enableTLS'].lower() == 'true': + protocol = 'https' + if 'application-properties' in services['configurations'] and 'atlas.server.https.port' in services['configurations']['application-properties']['properties']: + atlas_port = services['configurations']['application-properties']['properties']['atlas.server.https.port'] + else: + protocol = 'http' + if 'application-properties' in services['configurations'] and 'atlas.server.http.port' in services['configurations']['application-properties']['properties']: + atlas_port = services['configurations']['application-properties']['properties']['atlas.server.http.port'] + + atlas_rest_endpoint = '{0}://{1}:{2}'.format(protocol, atlas_host, atlas_port) + putTagsyncSiteProperty('ranger.tagsync.source.atlas', 'true') + putTagsyncSiteProperty('ranger.tagsync.source.atlasrest.endpoint', atlas_rest_endpoint) zookeeper_host_port = self.getZKHostPortString(services) if zookeeper_host_port and has_ranger_tagsync: - zookeeper_host_list = zookeeper_host_port.split(',') - putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_list[0]) + putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_port) if 'KAFKA' in servicesList and has_ranger_tagsync: kafka_hosts = self.getHostNamesWithComponent("KAFKA", "KAFKA_BROKER", services) http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 08b9554..6192e41 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -3912,6 +3912,14 @@ class TestHDP22StackAdvisor(TestCase): } } services = { + "services": + [ + { + "StackServices": { + "service_name" : "STORM" + } + } + ], "configurations": configurations } res_expected = [] @@ -3978,10 +3986,26 @@ class TestHDP22StackAdvisor(TestCase): recommendedDefaults = { "ranger-storm-plugin-enabled": "No", } - configurations = {} - services = {} + configurations = { + "cluster-env": { + "properties": { + "security_enabled": "false", + } + } + } + services = { + "services": + [ + { + "StackServices": { + "service_name" : "STORM" + } + } + ] + } # Test with ranger plugin enabled, validation fails res_expected = [{'config-type': 'ranger-env', 'message': 'Ranger Storm plugin should not be enabled in non-kerberos environment.', 'type': 'configuration', 'config-name': 'ranger-storm-plugin-enabled', 'level': 'WARN'}] + res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {}) self.assertEquals(res, res_expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad16dfd/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 4dfb8af..da0a704 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2110,3 +2110,35 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.getComponentHostNames = return_c6401_hostname self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + + def test_validateRangerConfigurationsEnv(self): + properties = { + "ranger-kafka-plugin-enabled": "Yes", + } + recommendedDefaults = { + "ranger-kafka-plugin-enabled": "No", + } + + configurations = { + "cluster-env": { + "properties": { + "security_enabled": "false", + } + } + } + services = { + "services": + [ + { + "StackServices": { + "service_name" : "KAFKA" + } + } + ] + } + + # Test with ranger plugin enabled, validation fails + res_expected = [{'config-type': 'ranger-env', 'message': 'Ranger Kafka plugin should not be enabled in non-kerberos environment.', 'type': 'configuration', 'config-name': 'ranger-kafka-plugin-enabled', 'level': 'WARN'}] + + res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {}) + self.assertEquals(res, res_expected) \ No newline at end of file
