Updated Branches: refs/heads/trunk dd0663dff -> 611988983
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/61198898/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java index d1bddd0..467cb8b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java @@ -877,22 +877,6 @@ public class GangliaPropertyProviderTest { {"metrics/disk", "disk_free", 92.428}, {"metrics/disk", "disk_total", 101.515}, {"metrics/disk", "part_max_used", 12.8}, - {"metrics/jvm", "gcCount", 0.0}, - {"metrics/jvm", "logError", 0.0}, - {"metrics/jvm", "logFatal", 0.0}, - {"metrics/jvm", "logInfo", 0.0}, - {"metrics/jvm", "logWarn", 0.0}, - {"metrics/jvm", "memMaxM", 910.25}, - {"metrics/jvm", "memHeapCommittedM", 48.8125}, - {"metrics/jvm", "memHeapUsedM", 9.5575423}, - {"metrics/jvm", "memNonHeapCommittedM", 23.5625}, - {"metrics/jvm", "memNonHeapUsedM", 22.78144}, - {"metrics/jvm", "threadsBlocked", 0.0}, - {"metrics/jvm", "threadsNew", 0.0}, - {"metrics/jvm", "threadsRunnable", 6.0}, - {"metrics/jvm", "threadsTerminated", 0.0}, - {"metrics/jvm", "threadsTimedWaiting", 3.0}, - {"metrics/jvm", "threadsWaiting", 9.0}, {"metrics/load", "load_fifteen", 0.026}, {"metrics/load", "load_five", 0.114}, {"metrics/load", "load_one", 0.226}, @@ -909,38 +893,6 @@ public class GangliaPropertyProviderTest { {"metrics/network", "pkts_out", 35.4}, {"metrics/process", "proc_run", 4.0}, {"metrics/process", "proc_total", 657.0}, - {"metrics/rpc", "numOpenConnections", 1.0}, - {"metrics/rpc", "receivedBytes", 6.34666666667}, - {"metrics/rpc", "rpcProcessingTime_avg_time", 2.0}, - {"metrics/rpc", "rpcProcessingTime_num_ops", 0.0266666666667}, - {"metrics/rpc", "rpcQueueTime_avg_time", 0.0}, - {"metrics/rpc", "rpcQueueTime_num_ops", 0.0266666666667}, - {"metrics/rpc", "sentBytes", 1.01333333333}, - {"metrics/rpc", "callQueueLength", 0.0}, - {"metrics/rpc", "rpcAuthenticationFailures", 0.0}, - {"metrics/rpc", "rpcAuthenticationSuccesses", 0.0}, - {"metrics/rpc", "rpcAuthorizationFailures", 0.0}, - {"metrics/rpc", "rpcAuthorizationSuccesses", 0.0266666666667}, - {"metrics/ugi", "loginFailure_avg_time", 0.0}, - {"metrics/ugi", "loginFailure_num_ops", 0.0}, - {"metrics/ugi", "loginSuccess_avg_time", 0.0}, - {"metrics/ugi", "loginSuccess_num_ops", 0.0}, - {"metrics/rpcdetailed", "acceptRecovery_avg_time", 0}, - {"metrics/rpcdetailed", "acceptRecovery_num_ops", 0}, - {"metrics/rpcdetailed", "finalizeLogSegment_avg_time", 0}, - {"metrics/rpcdetailed", "finalizeLogSegment_num_ops", 0}, - {"metrics/rpcdetailed", "getEditLogManifest_avg_time", 2.0}, - {"metrics/rpcdetailed", "getEditLogManifest_num_ops", 0.0266666666667}, - {"metrics/rpcdetailed", "getJournalState_avg_time", 0}, - {"metrics/rpcdetailed", "getJournalState_num_ops", 0}, - {"metrics/rpcdetailed", "journal_avg_time", 0}, - {"metrics/rpcdetailed", "journal_num_ops", 0}, - {"metrics/rpcdetailed", "newEpoch_avg_time", 0}, - {"metrics/rpcdetailed", "newEpoch_num_ops", 0}, - {"metrics/rpcdetailed", "prepareRecovery_avg_time", 0}, - {"metrics/rpcdetailed", "prepareRecovery_num_ops", 0}, - {"metrics/rpcdetailed", "startLogSegment_avg_time", 0}, - {"metrics/rpcdetailed", "startLogSegment_num_ops", 0}, {"metrics/dfs/journalNode", "batchesWritten", 0.0}, {"metrics/dfs/journalNode", "batchesWrittenWhileLagging", 0.0}, {"metrics/dfs/journalNode", "bytesWritten", 0.0}, @@ -979,6 +931,18 @@ public class GangliaPropertyProviderTest { Request request = PropertyHelper.getReadRequest(properties, temporalInfoMap); Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); + + Map<String, Object> p = PropertyHelper.getProperties(resource); + + for (String key : p.keySet()) { + if (!properties.contains(key)) { + System.out.printf(key); + } + } + + + + Assert.assertEquals(properties.size() + 2, PropertyHelper.getProperties(resource).size()); int i = 0; http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/61198898/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java index bd417df..26a0489 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java @@ -692,10 +692,10 @@ public class JMXPropertyProviderTest { // see test/resources/hdfs_journalnode_jmx.json for values Assert.assertEquals(1377795104272L, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "startTime"))); - Assert.assertEquals(954466304, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "heapMemoryMax"))); - Assert.assertEquals(14569736, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "heapMemoryUsed"))); - Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "nonHeapMemoryMax"))); - Assert.assertEquals(24993392, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "nonHeapMemoryUsed"))); + Assert.assertEquals(954466304, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax"))); + Assert.assertEquals(14569736, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed"))); + Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax"))); + Assert.assertEquals(24993392, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed"))); Assert.assertEquals(9100, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "gcCount"))); Assert.assertEquals(31641, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "gcTimeMillis"))); Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "logError"))); @@ -713,13 +713,13 @@ public class JMXPropertyProviderTest { Assert.assertEquals(3, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "threadsTimedWaiting"))); Assert.assertEquals(8, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "threadsWaiting"))); - Assert.assertEquals(1, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "numOpenConnections"))); - Assert.assertEquals(4928861, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "receivedBytes"))); - Assert.assertEquals(13.211112159230245, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "rpcProcessingTime_avg_time"))); - Assert.assertEquals(25067, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "rpcProcessingTime_num_ops"))); - Assert.assertEquals(0.19686821997924706, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "rpcQueueTime_avg_time"))); - Assert.assertEquals(25067, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "rpcQueueTime_num_ops"))); - Assert.assertEquals(6578899, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "sentBytes"))); + Assert.assertEquals(1, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "NumOpenConnections"))); + Assert.assertEquals(4928861, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "ReceivedBytes"))); + Assert.assertEquals(13.211112159230245, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "RpcProcessingTime_avg_time"))); + Assert.assertEquals(25067, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "RpcProcessingTime_num_ops"))); + Assert.assertEquals(0.19686821997924706, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "RpcQueueTime_avg_time"))); + Assert.assertEquals(25067, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "RpcQueueTime_num_ops"))); + Assert.assertEquals(6578899, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "SentBytes"))); Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "callQueueLen"))); Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "rpcAuthenticationFailures"))); Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "rpcAuthenticationSuccesses"))); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/61198898/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/PropertyHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/PropertyHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/PropertyHelperTest.java index 3ac7551..a22c96d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/PropertyHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/PropertyHelperTest.java @@ -23,9 +23,13 @@ import org.junit.Assert; import org.junit.Test; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; /** @@ -138,5 +142,277 @@ public class PropertyHelperTest { Assert.assertFalse(PropertyHelper.containsArguments("$X/foo/bar/$Y/baz/$Z")); } + + @Test + /** + * Test to make sure that point in time metrics are not in both JMX and Ganglia. + * A metric marked as point in time should not be available from both JMX + * and Ganglia. The preference is to get point in time metrics from JMX + * but they may come from Ganglia if not available from JMX. + * + * If there is a legitimate exception and the point in time metric should + * be available from both property providers then please add an exception to + * this test. + */ + public void testDuplicatePointInTimeMetrics() { + for (PropertyHelper.MetricsVersion version : PropertyHelper.MetricsVersion.values()) { + + TreeSet<String> set = new TreeSet<String>(); + + for (Resource.Type type : Resource.Type.values()) { + + Map<String, Map<String, PropertyInfo>> gids = + PropertyHelper.getGangliaPropertyIds(type, version); + + Map<String, Map<String, PropertyInfo>> jids = + PropertyHelper.getJMXPropertyIds(type, version); + + if (gids != null && jids != null) { + + gids = normalizeMetricNames(gids); + jids = normalizeMetricNames(jids); + + for (Map.Entry<String, Map<String, PropertyInfo>> gComponentEntry : gids.entrySet()) { + + String gComponent = gComponentEntry.getKey(); + + Set<Map.Entry<String, PropertyInfo>> gComponentEntries = gComponentEntry.getValue().entrySet(); + + for (Map.Entry<String, PropertyInfo> gMetricEntry : gComponentEntries) { + + Map<String, PropertyInfo> jMetrics = jids.get(gComponent); + + if (jMetrics != null) { + String gMetric = gMetricEntry.getKey(); + PropertyInfo jProperty = jMetrics.get(gMetric); + + if (jProperty != null) { + PropertyInfo gProperty = gMetricEntry.getValue(); + if (gProperty.isPointInTime()) { + String s = type + " : " + gComponent + " : " + gMetric + " : " + gProperty.getPropertyId(); + set.add(s); + } + } + } + } + } + } + } + if (set.size() > 0) { + System.out.println("The following point in time metrics are defined for both JMX and Ganglia."); + System.out.println("The preference is to get point in time metrics from JMX only if possible."); + System.out.println("If the metric can be obtained from JMX then set \"pointInTime\" : false for "); + System.out.println("the metric in the Ganglia properties definition, otherwise remove the metric "); + System.out.println("from the JMX properties definition.\n"); + System.out.println("Metric version :" + version + "\n"); + for (String s : set) { + System.out.println(s); + } + Assert.fail("Found duplicate point in time metrics."); + } + } + } + + @Test + /** + * Test to make sure that any metrics that are marked as temporal only in Ganglia are available + * as point in time from JMX. If a metric can not be provided by JMX it may be marked + * as point in time from Ganglia. + * + * If there is a legitimate exception and the metric should be temporal only then please add an + * exception to this test. + */ + public void testTemporalOnlyMetrics() { + + for (PropertyHelper.MetricsVersion version : PropertyHelper.MetricsVersion.values()) { + + TreeSet<String> set = new TreeSet<String>(); + + for (Resource.Type type : Resource.Type.values()) { + + Map<String, Map<String, PropertyInfo>> gids = + PropertyHelper.getGangliaPropertyIds(type, version); + + Map<String, Map<String, PropertyInfo>> jids = + PropertyHelper.getJMXPropertyIds(type, version); + + if (gids != null && jids != null) { + + gids = normalizeMetricNames(gids); + jids = normalizeMetricNames(jids); + + for (Map.Entry<String, Map<String, PropertyInfo>> gComponentEntry : gids.entrySet()) { + + String gComponent = gComponentEntry.getKey(); + + Set<Map.Entry<String, PropertyInfo>> gComponentEntries = gComponentEntry.getValue().entrySet(); + + for (Map.Entry<String, PropertyInfo> gMetricEntry : gComponentEntries) { + + Map<String, PropertyInfo> jMetrics = jids.get(gComponent); + + if (jMetrics != null) { + + String gMetric = gMetricEntry.getKey(); + PropertyInfo gProperty = gMetricEntry.getValue(); + + if (!gProperty.isPointInTime()) { + + PropertyInfo jProperty = jMetrics.get(gMetric); + + if (jProperty == null || !jProperty.isPointInTime()) { + + String s = type + " : " + gComponent + " : " + gMetric + " : " + gProperty.getPropertyId(); + + set.add(s); + } + } + } + } + } + } + } + if (set.size() > 0) { + System.out.println("The following metrics are marked as temporal only for Ganglia "); + System.out.println("but are not defined for JMX."); + System.out.println("The preference is to get point in time metrics from JMX if possible."); + System.out.println("If the metric can be obtained from JMX then add it to the JMX properties"); + System.out.println("definition, otherwise set set \"pointInTime\" : true for the metric in "); + System.out.println("the Ganglia properties definition.\n"); + System.out.println("Metric version :" + version + "\n"); + for (String s : set) { + System.out.println(s); + } + Assert.fail("Found temporal only metrics."); + } + } + } + + @Test + /** + * Test to make sure that no JMX metrics are marked as point in time. + */ + public void testJMXTemporal() { + + for (PropertyHelper.MetricsVersion version : PropertyHelper.MetricsVersion.values()) { + + TreeSet<String> set = new TreeSet<String>(); + + for (Resource.Type type : Resource.Type.values()) { + + Map<String, Map<String, PropertyInfo>> jids = + PropertyHelper.getJMXPropertyIds(type, version); + + if (jids != null) { + + for (Map.Entry<String, Map<String, PropertyInfo>> jComponentEntry : jids.entrySet()) { + + String jComponent = jComponentEntry.getKey(); + + Set<Map.Entry<String, PropertyInfo>> jComponentEntries = jComponentEntry.getValue().entrySet(); + for (Map.Entry<String, PropertyInfo> jMetricEntry : jComponentEntries) { + + String jMetric = jMetricEntry.getKey(); + + PropertyInfo jProperty = jMetricEntry.getValue(); + + if (jProperty.isTemporal()) { + String s = type + " : " + jComponent + " : " + jMetric + " : " + jProperty.getPropertyId(); + + set.add(s); + } + } + } + } + } + + if (set.size() > 0) { + System.out.println("The following metrics are marked as temporal JMX."); + System.out.println("JMX can provide point in time metrics only.\n"); + System.out.println("Metric version :" + version + "\n"); + for (String s : set) { + System.out.println(s); + } + Assert.fail("Found temporal JMX metrics."); + } + } + } + + @Test + /** + * Test to look for changes in case of the metric names + * (e.g. metrics/rpc/RpcQueueTime_num_ops to metrics/rpc/rpcQueueTime_num_ops). + */ + public void testMetricCaseChange() { + Map<String, Set<String>> metricSet = new HashMap<String, Set<String>>(); + for (int i = 0; i < 2; ++i) { + for (PropertyHelper.MetricsVersion version : PropertyHelper.MetricsVersion.values()) { + for (Resource.Type type : Resource.Type.values()) { + + Map<String, Map<String, PropertyInfo>> ids = i == 0 ? + PropertyHelper.getGangliaPropertyIds(type, version) : + PropertyHelper.getJMXPropertyIds(type, version); + + if (ids != null) { + + for (Map.Entry<String, Map<String, PropertyInfo>> gComponentEntry : ids.entrySet()) { + Set<Map.Entry<String, PropertyInfo>> gComponentEntries = gComponentEntry.getValue().entrySet(); + for (Map.Entry<String, PropertyInfo> gMetricEntry : gComponentEntries) { + + String gMetric = gMetricEntry.getKey(); + String key = gMetric.toLowerCase(); + + Set<String> metrics = metricSet.get(key); + if (metrics == null) { + metrics = new HashSet<String>(); + metricSet.put(key, metrics); + } + metrics.add(gMetric); + } + } + } + } + } + } + + List<Set<String>> failureList = new LinkedList<Set<String>>(); + for (Set<String> metrics : metricSet.values()) { + if (metrics.size() > 1) { + failureList.add(metrics); + } + } + + if (failureList.size() > 1) { + System.out.println("The following metrics differ only by case.\n"); + for (Set<String> metrics : failureList) { + System.out.println(" " + metrics); + } + Assert.fail("Found metric name case differences."); + } + } + + // remove any replacement tokens (e.g. $1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)) in the metric names + private static Map<String, Map<String, PropertyInfo>> normalizeMetricNames(Map<String, Map<String, PropertyInfo>> gids) { + + Map<String, Map<String, PropertyInfo>> returnMap = new HashMap<String, Map<String, PropertyInfo>>(); + + for (Map.Entry<String, Map<String, PropertyInfo>> gComponentEntry : gids.entrySet()) { + + String gComponent = gComponentEntry.getKey(); + Map<String, PropertyInfo> newMap = new HashMap<String, PropertyInfo>(); + + Set<Map.Entry<String, PropertyInfo>> gComponentEntries = gComponentEntry.getValue().entrySet(); + + for (Map.Entry<String, PropertyInfo> gMetricEntry : gComponentEntries) { + + String gMetric = gMetricEntry.getKey(); + PropertyInfo propertyInfo = gMetricEntry.getValue(); + + newMap.put(gMetric.replaceAll("\\$\\d+(\\.\\S+\\(\\S+\\))*", "*"), propertyInfo); + } + returnMap.put(gComponent, newMap); + } + return returnMap; + } }
