AMBARI-2910. Add various YARN time-series metrics. (ncole)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/83c4991e Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/83c4991e Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/83c4991e Branch: refs/heads/trunk Commit: 83c4991ec867ce816a3f68b9d1be4fe18a6e2f8b Parents: 0800104 Author: Nate Cole <[email protected]> Authored: Thu Aug 15 09:35:11 2013 -0400 Committer: Nate Cole <[email protected]> Committed: Thu Aug 15 11:33:06 2013 -0400 ---------------------------------------------------------------------- .../hadoop-metrics2.properties-GANGLIA.erb | 3 + .../templates/hadoop-metrics2.properties.erb | 3 + .../main/resources/ganglia_properties_2.json | 61 + .../ganglia/GangliaPropertyProviderTest.java | 42 +- .../temporal_ganglia_data_yarn_queues.txt | 15720 +++++++++++++++++ 5 files changed, 15828 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb index 3ab9875..206c64c 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb @@ -37,3 +37,6 @@ reducetask.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host" resourcemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8664 nodemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8660 historyserver.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8666 + +resourcemanager.sink.ganglia.tagsForPrefix.yarn=* +resourcemanager.sink.ganglia.tagsForPrefix.rpc=* http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb index 3ab9875..206c64c 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb @@ -37,3 +37,6 @@ reducetask.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host" resourcemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8664 nodemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8660 historyserver.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8666 + +resourcemanager.sink.ganglia.tagsForPrefix.yarn=* +resourcemanager.sink.ganglia.tagsForPrefix.rpc=* http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-server/src/main/resources/ganglia_properties_2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/ganglia_properties_2.json b/ambari-server/src/main/resources/ganglia_properties_2.json index 7dc2df1..5d4284c 100644 --- a/ambari-server/src/main/resources/ganglia_properties_2.json +++ b/ambari-server/src/main/resources/ganglia_properties_2.json @@ -6722,7 +6722,38 @@ "metric":"ugi.ugi.LoginFailureAvgTime", "pointInTime":false, "temporal":true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB":{ + "metric":"yarn.QueueMetrics.Queue=(.+).AvailableMB", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB":{ + "metric":"yarn.QueueMetrics.Queue=(.+).PendingMB", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB":{ + "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedMB", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers":{ + "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedContainers", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers":{ + "metric":"yarn.QueueMetrics.Queue=(.+).ReservedContainers", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers":{ + "metric":"yarn.QueueMetrics.Queue=(.+).PendingContainers", + "pointInTime" : false, + "temporal" : true } + }, "NODEMANAGER":{ "metrics/boottime":{ @@ -16877,6 +16908,36 @@ "metric":"ugi.ugi.LoginFailureAvgTime", "pointInTime":false, "temporal":true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB":{ + "metric":"yarn.QueueMetrics.Queue=(.+).AvailableMB", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB":{ + "metric":"yarn.QueueMetrics.Queue=(.+).PendingMB", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB":{ + "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedMB", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers":{ + "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedContainers", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers":{ + "metric":"yarn.QueueMetrics.Queue=(.+).ReservedContainers", + "pointInTime" : false, + "temporal" : true + }, + "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers":{ + "metric":"yarn.QueueMetrics.Queue=(.+).PendingContainers", + "pointInTime" : false, + "temporal" : true } }, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java index d476f04..1fc1e02 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java @@ -56,7 +56,10 @@ public class GangliaPropertyProviderTest { private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "cluster_name"); private static final String HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name"); private static final String COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "component_name"); - + + private static final String RM_CATEGORY_1 = "metrics/yarn/Queue/root/default"; + private static final String RM_AVAILABLE_MEMORY_PROPERTY = PropertyHelper.getPropertyId(RM_CATEGORY_1, "AvailableMB"); + private ComponentSSLConfiguration configuration; @Parameterized.Parameters @@ -534,6 +537,43 @@ public class GangliaPropertyProviderTest { Assert.assertEquals(11, PropertyHelper.getProperties(resource).size()); Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY)); } + + @Test + public void testPopulateResources_params_category5() throws Exception { + TestStreamProvider streamProvider = new TestStreamProvider("temporal_ganglia_data_yarn_queues.txt"); + TestGangliaHostProvider hostProvider = new TestGangliaHostProvider(); + + GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider( + PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP2), + streamProvider, + configuration, + hostProvider, + CLUSTER_NAME_PROPERTY_ID, + HOST_NAME_PROPERTY_ID, + COMPONENT_NAME_PROPERTY_ID); + + Resource resource = new ResourceImpl(Resource.Type.HostComponent); + + resource.setProperty(HOST_NAME_PROPERTY_ID, "dev01.ambari.apache.org"); + resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "RESOURCEMANAGER"); + + // only ask for one property + Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); + temporalInfoMap.put(RM_CATEGORY_1, new TemporalInfoImpl(10L, 20L, 1L)); + + Request request = PropertyHelper.getReadRequest(Collections.singleton(RM_CATEGORY_1), temporalInfoMap); + + Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); + + String expected = (configuration.isGangliaSSL() ? "https" : "http") + + "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPResourceManager&h=dev01.ambari.apache.org&s=10&e=20&r=1"; + Assert.assertEquals(expected, streamProvider.getLastSpec()); + + System.out.println(PropertyHelper.getProperties(resource)); + + Assert.assertEquals(14, PropertyHelper.getProperties(resource).size()); + Assert.assertNotNull(resource.getPropertyValue(RM_AVAILABLE_MEMORY_PROPERTY)); + } private static class TestGangliaHostProvider implements GangliaHostProvider {
