Repository: ambari Updated Branches: refs/heads/trunk 4eb2be46e -> 629a893aa
Revert "AMBARI-17415: Ambari configuration for ranger-tagsync needs to support property for atlas keystore filename (Mugdha Varadkar via jluniya)" Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/629a893a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/629a893a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/629a893a Branch: refs/heads/trunk Commit: 629a893aa287ab4fd9a33f69d9101e42e8d182b9 Parents: 4eb2be4 Author: Andrii Tkach <atk...@apache.org> Authored: Thu Jul 7 18:57:51 2016 +0300 Committer: Andrii Tkach <atk...@apache.org> Committed: Thu Jul 7 18:57:51 2016 +0300 ---------------------------------------------------------------------- .../0.6.0/configuration/ranger-tagsync-site.xml | 31 +------------------ .../stacks/HDP/2.2/services/stack_advisor.py | 31 ++++++++----------- .../stacks/HDP/2.3/services/stack_advisor.py | 29 ++---------------- .../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 7 ----- .../configuration/ranger-tagsync-site.xml | 9 ------ .../stacks/HDP/2.5/services/stack_advisor.py | 21 ++----------- .../stacks/2.2/common/test_stack_advisor.py | 28 ++--------------- .../stacks/2.3/common/test_stack_advisor.py | 32 -------------------- 8 files changed, 20 insertions(+), 168 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml index 73b8227..7985f58 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml @@ -88,7 +88,7 @@ <property> <name>ranger.tagsync.source.atlasrest.download.interval.millis</name> <display-name>AtlasREST Source: Atlas source download interval</display-name> - <value>60000</value> + <value/> <value-attributes> <empty-value-valid>true</empty-value-valid> </value-attributes> @@ -137,20 +137,6 @@ <empty-value-valid>true</empty-value-valid> </value-attributes> <on-ambari-upgrade add="true"/> - <depends-on> - <property> - <type>application-properties</type> - <name>atlas.server.http.port</name> - </property> - <property> - <type>application-properties</type> - <name>atlas.server.https.port</name> - </property> - <property> - <type>application-properties</type> - <name>atlas.enableTLS</name> - </property> - </depends-on> </property> <property> <name>ranger.tagsync.kerberos.principal</name> @@ -176,19 +162,4 @@ <description/> <on-ambari-upgrade add="true"/> </property> - <property> - <name>ranger.tagsync.source.atlasrest.keystore.filename</name> - <value>/etc/ranger/tagsync/conf/atlasuser.jceks</value> - <description>Tagsync atlasrest keystore file</description> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>ranger.tagsync.source.atlasrest.username</name> - <value>admin</value> - <description/> - <on-ambari-upgrade add="true"/> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 4e99333..2c5eed3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -229,7 +229,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): keyserverHostsString = services["configurations"]["hadoop-env"]["properties"]["keyserver_host"] keyserverPortString = services["configurations"]["hadoop-env"]["properties"]["keyserver_port"] - # Irrespective of what hadoop-env has, if Ranger-KMS is installed, we use its values. + # Irrespective of what hadoop-env has, if Ranger-KMS is installed, we use its values. rangerKMSServerHosts = self.getHostsWithComponent("RANGER_KMS", "RANGER_KMS_SERVER", services, hosts) if rangerKMSServerHosts is not None and len(rangerKMSServerHosts) > 0: rangerKMSServerHostsArray = [] @@ -1134,7 +1134,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): validationItems.append({"config-name" : address_property, "item" : self.getErrorItem(address_property + " does not contain a valid host:port authority: " + value)}) - #Adding Ranger Plugin logic here + #Adding Ranger Plugin logic here ranger_plugin_properties = getSiteProperties(configurations, "ranger-hdfs-plugin-properties") ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties else 'No' servicesList = [service["StackServices"]["service_name"] for service in services["services"]] @@ -1237,8 +1237,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts): hive_server2 = properties - validationItems = [] - #Adding Ranger Plugin logic here + validationItems = [] + #Adding Ranger Plugin logic here ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties") hive_env_properties = getSiteProperties(configurations, "hive-env") ranger_plugin_enabled = 'hive_security_authorization' in hive_env_properties and hive_env_properties['hive_security_authorization'].lower() == 'ranger' @@ -1376,7 +1376,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "item": self.getWarnItem( "{0} and {1} sum should not exceed {2}".format(prop_name1, prop_name2, props_max_sum))}) - #Adding Ranger Plugin logic here + #Adding Ranger Plugin logic here ranger_plugin_properties = getSiteProperties(configurations, "ranger-hbase-plugin-properties") ranger_plugin_enabled = ranger_plugin_properties['ranger-hbase-plugin-enabled'] if ranger_plugin_properties else 'No' prop_name = 'hbase.security.authorization' @@ -1430,7 +1430,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): "item": self.getWarnItem( "If bucketcache ioengine is enabled, {0} should be set".format(prop_name3))}) - # Validate hbase.security.authentication. + # Validate hbase.security.authentication. # Kerberos works only when security enabled. if "hbase.security.authentication" in properties: hbase_security_kerberos = properties["hbase.security.authentication"].lower() == "kerberos" @@ -1505,7 +1505,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): validationItems = [] ranger_plugin_properties = getSiteProperties(configurations, "ranger-storm-plugin-properties") ranger_plugin_enabled = ranger_plugin_properties['ranger-storm-plugin-enabled'] if ranger_plugin_properties else 'No' - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] if ranger_plugin_enabled.lower() == 'yes': # ranger-hdfs-plugin must be enabled in ranger-env ranger_env = getServicesSiteProperties(services, 'ranger-env') @@ -1514,11 +1513,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): validationItems.append({"config-name": 'ranger-storm-plugin-enabled', "item": self.getWarnItem( "ranger-storm-plugin-properties/ranger-storm-plugin-enabled must correspond ranger-env/ranger-storm-plugin-enabled")}) - if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()) and not 'KERBEROS' in servicesList: - validationItems.append({"config-name": "ranger-storm-plugin-enabled", - "item": self.getWarnItem( - "Ranger Storm plugin should not be enabled in non-kerberos environment.")}) - return self.toConfigurationValidationProblems(validationItems, "ranger-storm-plugin-properties") def validateYARNEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): @@ -1552,12 +1546,13 @@ class HDP22StackAdvisor(HDP21StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "ranger-yarn-plugin-properties") def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts): - ranger_env_properties = properties validationItems = [] - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - if "ranger-storm-plugin-enabled" in ranger_env_properties and ranger_env_properties['ranger-storm-plugin-enabled'].lower() == 'yes' and not 'KERBEROS' in servicesList: - validationItems.append({"config-name": "ranger-storm-plugin-enabled", - "item": self.getWarnItem("Ranger Storm plugin should not be enabled in non-kerberos environment.")}) + if "ranger-storm-plugin-enabled" in properties and "ranger-storm-plugin-enabled" in recommendedDefaults and \ + properties["ranger-storm-plugin-enabled"] != recommendedDefaults["ranger-storm-plugin-enabled"]: + validationItems.append({"config-name": "ranger-storm-plugin-enabled", + "item": self.getWarnItem( + "Ranger Storm plugin should not be enabled in non-kerberos environment.")}) + return self.toConfigurationValidationProblems(validationItems, "ranger-env") def getMastersWithMultipleInstances(self): @@ -1578,7 +1573,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def getAffectedConfigs(self, services): affectedConfigs = super(HDP22StackAdvisor, self).getAffectedConfigs(services) - # There are configs that are not defined in the stack but added/removed by + # There are configs that are not defined in the stack but added/removed by # stack-advisor. Here we add such configs in order to clear the config # filtering down in base class configsList = [affectedConfig["type"] + "/" + affectedConfig["name"] for affectedConfig in affectedConfigs] http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 460aea3..879008b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -724,8 +724,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "HBASE": {"hbase-site": self.validateHBASEConfigurations}, "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations}, "YARN": {"yarn-site": self.validateYARNConfigurations}, - "RANGER": {"admin-properties": self.validateRangerAdminConfigurations, - "ranger-env": self.validateRangerConfigurationsEnv} + "RANGER": {"admin-properties": self.validateRangerAdminConfigurations} } self.mergeValidators(parentValidators, childValidators) return parentValidators @@ -908,11 +907,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "If Ranger Kafka Plugin is enabled."\ "{0} needs to be set to {1}".format(prop_name,prop_val))}) - if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()) and not 'KERBEROS' in servicesList: - validationItems.append({"config-name": "ranger-kafka-plugin-enabled", - "item": self.getWarnItem( - "Ranger Kafka plugin should not be enabled in non-kerberos environment.")}) - return self.toConfigurationValidationProblems(validationItems, "kafka-broker") def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): @@ -923,7 +917,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled'] if yarn_resource_proxy_enabled.lower() == 'true': validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled', - "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled " \ + "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled "\ "property value as false under yarn-site" )}) @@ -943,22 +937,3 @@ class HDP23StackAdvisor(HDP22StackAdvisor): 'item':self.getWarnItem('Ranger External URL should not contain trailing slash "/"')}) return self.toConfigurationValidationProblems(validationItems,'admin-properties') - def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts): - parentValidationProblems = super(HDP23StackAdvisor, self).validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, hosts) - ranger_env_properties = properties - validationItems = [] - security_enabled = False - - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - if 'KERBEROS' in servicesList: - security_enabled = True - - if "ranger-kafka-plugin-enabled" in ranger_env_properties and ranger_env_properties["ranger-kafka-plugin-enabled"].lower() == 'yes' and not security_enabled: - validationItems.append({"config-name": "ranger-kafka-plugin-enabled", - "item": self.getWarnItem( - "Ranger Kafka plugin should not be enabled in non-kerberos environment.")}) - - validationProblems = self.toConfigurationValidationProblems(validationItems, "ranger-env") - validationProblems.extend(parentValidationProblems) - return validationProblems - http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml index 6e8308a..020e339 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml @@ -62,13 +62,6 @@ </osSpecific> </osSpecifics> - <configuration-dependencies> - <config-type>ranger-atlas-audit</config-type> - <config-type>ranger-atlas-plugin-properties</config-type> - <config-type>ranger-atlas-policymgr-ssl</config-type> - <config-type>ranger-atlas-security</config-type> - </configuration-dependencies> - <requiredServices> <service>KAFKA</service> </requiredServices> http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml index 6a0991b..c3fe932 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml @@ -34,13 +34,4 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> - <property> - <name>ranger.tagsync.source.atlasrest.keystore.filename</name> - <value>/usr/hdp/current/ranger-tagsync/conf/atlasuser.jceks</value> - <description>Tagsync atlasrest keystore file</description> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 2ca8c05..1d092cd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1399,29 +1399,12 @@ class HDP25StackAdvisor(HDP24StackAdvisor): has_ranger_tagsync = len(ranger_tagsync_host) > 0 if 'ATLAS' in servicesList and has_ranger_tagsync: - atlas_hosts = self.getHostNamesWithComponent("ATLAS", "ATLAS_SERVER", services) - atlas_host = 'localhost' if len(atlas_hosts) == 0 else atlas_hosts[0] - protocol = 'http' - atlas_port = '21000' - - if 'application-properties' in services['configurations'] and 'atlas.enableTLS' in services['configurations']['application-properties']['properties'] \ - and services['configurations']['application-properties']['properties']['atlas.enableTLS'].lower() == 'true': - protocol = 'https' - if 'application-properties' in services['configurations'] and 'atlas.server.https.port' in services['configurations']['application-properties']['properties']: - atlas_port = services['configurations']['application-properties']['properties']['atlas.server.https.port'] - else: - protocol = 'http' - if 'application-properties' in services['configurations'] and 'atlas.server.http.port' in services['configurations']['application-properties']['properties']: - atlas_port = services['configurations']['application-properties']['properties']['atlas.server.http.port'] - - atlas_rest_endpoint = '{0}://{1}:{2}'.format(protocol, atlas_host, atlas_port) - putTagsyncSiteProperty('ranger.tagsync.source.atlas', 'true') - putTagsyncSiteProperty('ranger.tagsync.source.atlasrest.endpoint', atlas_rest_endpoint) zookeeper_host_port = self.getZKHostPortString(services) if zookeeper_host_port and has_ranger_tagsync: - putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_port) + zookeeper_host_list = zookeeper_host_port.split(',') + putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_list[0]) if 'KAFKA' in servicesList and has_ranger_tagsync: kafka_hosts = self.getHostNamesWithComponent("KAFKA", "KAFKA_BROKER", services) http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 9443042..54edf1d 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -3932,14 +3932,6 @@ class TestHDP22StackAdvisor(TestCase): } } services = { - "services": - [ - { - "StackServices": { - "service_name" : "STORM" - } - } - ], "configurations": configurations } res_expected = [] @@ -4006,26 +3998,10 @@ class TestHDP22StackAdvisor(TestCase): recommendedDefaults = { "ranger-storm-plugin-enabled": "No", } - configurations = { - "cluster-env": { - "properties": { - "security_enabled": "false", - } - } - } - services = { - "services": - [ - { - "StackServices": { - "service_name" : "STORM" - } - } - ] - } + configurations = {} + services = {} # Test with ranger plugin enabled, validation fails res_expected = [{'config-type': 'ranger-env', 'message': 'Ranger Storm plugin should not be enabled in non-kerberos environment.', 'type': 'configuration', 'config-name': 'ranger-storm-plugin-enabled', 'level': 'WARN'}] - res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {}) self.assertEquals(res, res_expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/629a893a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 784b271..2d98558 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2110,35 +2110,3 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.getComponentHostNames = return_c6401_hostname self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - - def test_validateRangerConfigurationsEnv(self): - properties = { - "ranger-kafka-plugin-enabled": "Yes", - } - recommendedDefaults = { - "ranger-kafka-plugin-enabled": "No", - } - - configurations = { - "cluster-env": { - "properties": { - "security_enabled": "false", - } - } - } - services = { - "services": - [ - { - "StackServices": { - "service_name" : "KAFKA" - } - } - ] - } - - # Test with ranger plugin enabled, validation fails - res_expected = [{'config-type': 'ranger-env', 'message': 'Ranger Kafka plugin should not be enabled in non-kerberos environment.', 'type': 'configuration', 'config-name': 'ranger-kafka-plugin-enabled', 'level': 'WARN'}] - - res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {}) - self.assertEquals(res, res_expected) \ No newline at end of file
