AMBARI-18857 : Add Service wizard: Storm site configs related to Ambari metrics service are not displayed on ui. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ef91365b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ef91365b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ef91365b Branch: refs/heads/branch-feature-AMBARI-18456 Commit: ef91365b7c1e294ac42006da71c647012679dbe6 Parents: 6e314a6 Author: Aravindan Vijayan <[email protected]> Authored: Mon Nov 21 13:01:24 2016 -0800 Committer: Aravindan Vijayan <[email protected]> Committed: Mon Nov 21 13:01:24 2016 -0800 ---------------------------------------------------------------------- .../0.1.0/configuration/storm-site.xml | 84 -------------------- .../AMBARI_METRICS/0.1.0/metainfo.xml | 4 - .../STORM/1.0.1/configuration/storm-site.xml | 54 +++++++++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 42 +++++++++- .../stacks/2.5/common/test_stack_advisor.py | 76 ++++++++++++++++++ 5 files changed, 171 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ef91365b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml deleted file mode 100644 index 7059b90..0000000 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml +++ /dev/null @@ -1,84 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="true"> - <property> - <name>metrics.reporter.register</name> - <value>org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter</value> - <description>Topology metrics reporter.</description> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>storm.cluster.metrics.consumer.register</name> - <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]</value> - <description></description> - <value-attributes> - <overridable>false</overridable> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>topology.metrics.consumer.register</name> - <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]</value> - <description></description> - <value-attributes> - <overridable>false</overridable> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>topology.metrics.aggregate.per.worker</name> - <value>true</value> - <description></description> - <value-attributes> - <overridable>false</overridable> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>topology.metrics.aggregate.metric.evict.secs</name> - <value>5</value> - <description></description> - <value-attributes> - <overridable>false</overridable> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>topology.metrics.expand.map.type</name> - <value>true</value> - <description></description> - <value-attributes> - <overridable>false</overridable> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>topology.metrics.metric.name.separator</name> - <value>.</value> - <description></description> - <value-attributes> - <overridable>false</overridable> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef91365b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index d2f5862..43a4320 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -209,10 +209,6 @@ <config-type>ams-env</config-type> </configuration-dependencies> - <excluded-config-types> - <config-type>storm-site</config-type> - </excluded-config-types> - <quickLinksConfigurations> <quickLinksConfiguration> <fileName>quicklinks.json</fileName> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef91365b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml index 077590a..558beaf 100644 --- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml @@ -117,4 +117,58 @@ <on-ambari-upgrade add="false"/> <deleted>true</deleted> </property> + <property> + <name>storm.cluster.metrics.consumer.register</name> + <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]</value> + <description></description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>topology.metrics.consumer.register</name> + <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]</value> + <description></description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>topology.metrics.aggregate.per.worker</name> + <value>true</value> + <description></description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>topology.metrics.aggregate.metric.evict.secs</name> + <value>5</value> + <description></description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>topology.metrics.expand.map.type</name> + <value>true</value> + <description></description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>topology.metrics.metric.name.separator</name> + <value>.</value> + <description></description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/ef91365b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index dfb7b0c..519602b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -117,11 +117,36 @@ class HDP25StackAdvisor(HDP24StackAdvisor): "YARN": {"yarn-site": self.validateYarnConfigurations}, "RANGER": {"ranger-tagsync-site": self.validateRangerTagsyncConfigurations}, "SPARK2": {"spark2-defaults": self.validateSpark2Defaults, - "spark2-thrift-sparkconf": self.validateSpark2ThriftSparkConf} + "spark2-thrift-sparkconf": self.validateSpark2ThriftSparkConf}, + "STORM": {"storm-site": self.validateStormConfigurations}, } self.mergeValidators(parentValidators, childValidators) return parentValidators + def validateStormConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + super(HDP25StackAdvisor, self).validateStormConfigurations(properties, recommendedDefaults, configurations, services, hosts) + validationItems = [] + + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + # Storm AMS integration + if 'AMBARI_METRICS' in servicesList: + if "storm.cluster.metrics.consumer.register" in properties and \ + 'null' in properties.get("storm.cluster.metrics.consumer.register"): + + validationItems.append({"config-name": 'storm.cluster.metrics.consumer.register', + "item": self.getWarnItem( + "Should be set to recommended value to report metrics to Ambari Metrics service.")}) + + if "topology.metrics.consumer.register" in properties and \ + 'null' in properties.get("topology.metrics.consumer.register"): + + validationItems.append({"config-name": 'topology.metrics.consumer.register', + "item": self.getWarnItem( + "Should be set to recommended value to report metrics to Ambari Metrics service.")}) + + + return self.toConfigurationValidationProblems(validationItems, "storm-site") + def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): application_properties = getSiteProperties(configurations, "application-properties") validationItems = [] @@ -455,6 +480,21 @@ class HDP25StackAdvisor(HDP24StackAdvisor): else: putStormSiteAttributes('nimbus.authorizer', 'delete', 'true') + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + # Storm AMS integration + if 'AMBARI_METRICS' in servicesList: + putStormSiteProperty('storm.cluster.metrics.consumer.register', '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]') + putStormSiteProperty('topology.metrics.consumer.register', + '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink", ' + '"parallelism.hint": 1, ' + '"whitelist": ["kafkaOffset\\\..+/", "__complete-latency", "__process-latency", ' + '"__receive\\\.population$", "__sendqueue\\\.population$", "__execute-count", "__emit-count", ' + '"__ack-count", "__fail-count", "memory/heap\\\.usedBytes$", "memory/nonHeap\\\.usedBytes$", ' + '"GC/.+\\\.count$", "GC/.+\\\.timeMs$"]}]') + else: + putStormSiteProperty('storm.cluster.metrics.consumer.register', 'null') + putStormSiteProperty('topology.metrics.consumer.register', 'null') + def constructAtlasRestAddress(self, services, hosts): """ :param services: Collection of services in the cluster with configs http://git-wip-us.apache.org/repos/asf/ambari/blob/ef91365b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 7124307..7759c09 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -7657,6 +7657,75 @@ class TestHDP25StackAdvisor(TestCase): # Test nimbus.authorizer with Ranger Storm plugin disabled in non-kerberos environment self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations['storm-site']['property_attributes']['nimbus.authorizer'], {'delete': 'true'}, "Test nimbus.authorizer with Ranger Storm plugin disabled in non-kerberos environment") + self.assertEquals(configurations['storm-site']['properties']['storm.cluster.metrics.consumer.register'], 'null') + self.assertEquals(configurations['storm-site']['properties']['topology.metrics.consumer.register'], 'null') + + services = { + "services": + [ + { + "StackServices": { + "service_name" : "STORM", + "service_version" : "1.0.1.0.0" + } + }, + { + "StackServices": { + "service_name": "RANGER", + "service_version": "0.6.0" + + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "RANGER_ADMIN", + "hostnames": ["host1"] + } + } + ] + }, + { + "StackServices": { + "service_name": "AMBARI_METRICS" + }, + "components": [{ + "StackServiceComponents": { + "component_name": "METRICS_COLLECTOR", + "hostnames": ["host1"] + } + + }, { + "StackServiceComponents": { + "component_name": "METRICS_MONITOR", + "hostnames": ["host1"] + } + + }] + } + ], + "Versions": { + "stack_version": "2.5" + }, + "configurations": { + "storm-site": { + "properties": { + "nimbus.authorizer" : "org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer", + "nimbus.impersonation.acl" :"{{{storm_bare_jaas_principal}} : {hosts: ['*'], groups: ['*']}}" + }, + "property_attributes": {} + }, + "storm-env": { + "properties":{ + "storm_principal_name": "[email protected]" + }, + }, + "ranger-storm-plugin-properties": { + "properties": { + "ranger-storm-plugin-enabled": "No" + } + } + } + } # Test nimbus.authorizer with Ranger Storm plugin enabled in non-kerberos environment configurations['storm-site']['properties'] = {} @@ -7664,6 +7733,13 @@ class TestHDP25StackAdvisor(TestCase): services['configurations']['ranger-storm-plugin-properties']['properties']['ranger-storm-plugin-enabled'] = 'Yes' self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations['storm-site']['property_attributes']['nimbus.authorizer'], {'delete': 'true'}, "Test nimbus.authorizer with Ranger Storm plugin enabled in non-kerberos environment") + self.assertEquals(configurations['storm-site']['properties']['storm.cluster.metrics.consumer.register'], '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]') + self.assertEquals(configurations['storm-site']['properties']['topology.metrics.consumer.register'], '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink", ' + '"parallelism.hint": 1, ' + '"whitelist": ["kafkaOffset\\\..+/", "__complete-latency", "__process-latency", ' + '"__receive\\\.population$", "__sendqueue\\\.population$", "__execute-count", "__emit-count", ' + '"__ack-count", "__fail-count", "memory/heap\\\.usedBytes$", "memory/nonHeap\\\.usedBytes$", ' + '"GC/.+\\\.count$", "GC/.+\\\.timeMs$"]}]') # Test nimbus.authorizer with Ranger Storm plugin being enabled in kerberos environment configurations['storm-site']['properties'] = {}
