Updated Branches: refs/heads/trunk a1a72c62a -> db1df4582
AMBARI-2786. YARN time series data needed for NodeManager statuses. (Andrew Onischuk via smohanty) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/db1df458 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/db1df458 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/db1df458 Branch: refs/heads/trunk Commit: db1df4582b0024df9ad62c3ea1208fa889407668 Parents: a1a72c6 Author: Sumit Mohanty <[email protected]> Authored: Wed Jul 31 12:57:27 2013 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Jul 31 12:57:27 2013 -0700 ---------------------------------------------------------------------- .../main/resources/ganglia_properties_2.json | 50 ++++++++++++++++++++ .../src/main/resources/jmx_properties_2.json | 50 ++++++++++++++++++++ .../ganglia/GangliaPropertyProviderTest.java | 47 ++++++++++++++++++ .../controller/jmx/JMXPropertyProviderTest.java | 6 +++ 4 files changed, 153 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/db1df458/ambari-server/src/main/resources/ganglia_properties_2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/ganglia_properties_2.json b/ambari-server/src/main/resources/ganglia_properties_2.json index b78210c..525463c 100644 --- a/ambari-server/src/main/resources/ganglia_properties_2.json +++ b/ambari-server/src/main/resources/ganglia_properties_2.json @@ -6578,6 +6578,31 @@ "pointInTime":false, "temporal":true }, + "metrics/yarn/ClusterMetrics/NumActiveNMs":{ + "metric":"yarn.ClusterMetrics.NumActiveNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{ + "metric":"yarn.ClusterMetrics.NumDecommissionedNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumLostNMs":{ + "metric":"yarn.ClusterMetrics.NumLostNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{ + "metric":"yarn.ClusterMetrics.NumUnhealthyNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumRebootedNMs":{ + "metric":"yarn.ClusterMetrics.NumRebootedNMs", + "pointInTime":false, + "temporal":true + }, "metrics/yarn/Queue/AllocatedVCores":{ "metric":"yarn.QueueMetrics.AllocatedVCores", "pointInTime":false, @@ -16698,6 +16723,31 @@ "pointInTime":false, "temporal":true }, + "metrics/yarn/ClusterMetrics/NumActiveNMs":{ + "metric":"yarn.ClusterMetrics.NumActiveNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{ + "metric":"yarn.ClusterMetrics.NumDecommissionedNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumLostNMs":{ + "metric":"yarn.ClusterMetrics.NumLostNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{ + "metric":"yarn.ClusterMetrics.NumUnhealthyNMs", + "pointInTime":false, + "temporal":true + }, + "metrics/yarn/ClusterMetrics/NumRebootedNMs":{ + "metric":"yarn.ClusterMetrics.NumRebootedNMs", + "pointInTime":false, + "temporal":true + }, "metrics/yarn/Queue/AllocatedVCores":{ "metric":"yarn.QueueMetrics.AllocatedVCores", "pointInTime":false, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/db1df458/ambari-server/src/main/resources/jmx_properties_2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/jmx_properties_2.json b/ambari-server/src/main/resources/jmx_properties_2.json index 3eafb0e..2e8e93a 100644 --- a/ambari-server/src/main/resources/jmx_properties_2.json +++ b/ambari-server/src/main/resources/jmx_properties_2.json @@ -3560,6 +3560,31 @@ "metric" : "Hadoop:service=ResourceManager,name=RMNMInfo.LiveNodeManagers", "pointInTime" : true, "temporal" : false + }, + "metrics/yarn/ClusterMetrics/NumActiveNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumLostNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumRebootedNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs", + "pointInTime":true, + "temporal":false }, "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60":{ "metric" : "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_60", @@ -7914,6 +7939,31 @@ "pointInTime" : true, "temporal" : false }, + "metrics/yarn/ClusterMetrics/NumActiveNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumLostNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs", + "pointInTime":true, + "temporal":false + }, + "metrics/yarn/ClusterMetrics/NumRebootedNMs":{ + "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs", + "pointInTime":true, + "temporal":false + }, "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60":{ "metric" : "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_60", "pointInTime" : true, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/db1df458/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java index 7f57827..d476f04 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java @@ -23,6 +23,7 @@ import org.apache.ambari.server.controller.internal.ResourceImpl; import org.apache.ambari.server.controller.internal.TemporalInfoImpl; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.TemporalInfo; import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.apache.ambari.server.controller.utilities.PropertyHelper.MetricsVersion; @@ -238,6 +239,52 @@ public class GangliaPropertyProviderTest { Assert.assertEquals(expected, streamProvider.getLastSpec()); } + + class PopulateResourceManagerResourcesTest{ + public void testProperty(String property, String jmxMetricName) throws SystemException{ + TestStreamProvider streamProvider = new TestStreamProvider("yarn_ganglia_data.txt"); + TestGangliaHostProvider hostProvider = new TestGangliaHostProvider(); + + GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider( + PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP2), + streamProvider, + configuration, + hostProvider, + CLUSTER_NAME_PROPERTY_ID, + HOST_NAME_PROPERTY_ID, + COMPONENT_NAME_PROPERTY_ID); + + // flume + Resource resource = new ResourceImpl(Resource.Type.HostComponent); + + resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal"); + resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "RESOURCEMANAGER"); + + // only ask for one property + Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); + temporalInfoMap.put(property, new TemporalInfoImpl(10L, 20L, 1L)); + Request request = PropertyHelper.getReadRequest(Collections.singleton(property), temporalInfoMap); + + Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); + + String expected = (configuration.isGangliaSSL() ? "https" : "http") + + String.format("://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPResourceManager&h=ip-10-39-113-33.ec2.internal&m=%s&s=10&e=20&r=1",jmxMetricName); + Assert.assertEquals(expected, streamProvider.getLastSpec()); + + Assert.assertEquals(3, PropertyHelper.getProperties(resource).size()); + Assert.assertNotNull(resource.getPropertyValue(property)); + } + } + + @Test + public void testPopulateResources_resourcemanager_clustermetrics() throws Exception { + PopulateResourceManagerResourcesTest tester = new PopulateResourceManagerResourcesTest(); + tester.testProperty("metrics/yarn/ClusterMetrics/NumActiveNMs","yarn.ClusterMetrics.NumActiveNMs"); + tester.testProperty("metrics/yarn/ClusterMetrics/NumDecommissionedNMs","yarn.ClusterMetrics.NumDecommissionedNMs"); + tester.testProperty("metrics/yarn/ClusterMetrics/NumLostNMs", "yarn.ClusterMetrics.NumLostNMs"); + tester.testProperty("metrics/yarn/ClusterMetrics/NumUnhealthyNMs", "yarn.ClusterMetrics.NumUnhealthyNMs"); + tester.testProperty("metrics/yarn/ClusterMetrics/NumRebootedNMs", "yarn.ClusterMetrics.NumRebootedNMs"); + } @Test http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/db1df458/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java index 849869c..6571267 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java @@ -375,6 +375,12 @@ public class JMXPropertyProviderTest { Assert.assertEquals(8192, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/Queue/root", "AvailableMB"))); Assert.assertEquals(1, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/Queue/root", "AvailableVCores"))); Assert.assertEquals(2, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/Queue/root", "AppsSubmitted"))); + + Assert.assertEquals(1, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumActiveNMs"))); + Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumDecommissionedNMs"))); + Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumLostNMs"))); + Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumUnhealthyNMs"))); + Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumRebootedNMs"))); } @Test
