Updated Branches: refs/heads/trunk 9496abdd2 -> def7e8cf9
AMBARI-3111. YARN: Specific queue time-series call gives data for all queues (Dmitry Sen via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/def7e8cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/def7e8cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/def7e8cf Branch: refs/heads/trunk Commit: def7e8cf9e26c89b12c2391ad60ccb063b65f3d9 Parents: 9496abd Author: Lisnichenko Dmitro <[email protected]> Authored: Thu Sep 5 19:46:11 2013 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Thu Sep 5 19:46:11 2013 +0300 ---------------------------------------------------------------------- .../ganglia/GangliaPropertyProvider.java | 7 +++ .../internal/AbstractPropertyProvider.java | 35 ++++++++++++ .../controller/internal/BaseProvider.java | 31 ++++++++--- .../ganglia/GangliaPropertyProviderTest.java | 2 +- .../internal/AbstractPropertyProviderTest.java | 56 ++++++++++++++++++++ .../controller/internal/BaseProviderTest.java | 20 +++++++ 6 files changed, 144 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/def7e8cf/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java index 5aedfc9..1482330 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java @@ -218,6 +218,13 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider { for (String id : ids) { Map<String, PropertyInfo> propertyInfoMap = new HashMap<String, PropertyInfo>(); + Map<String, PropertyInfo> componentMetricMap = + getComponentMetrics().get(getComponentName(resource)); + + if (!componentMetricMap.containsKey(id)) { + updateComponentMetricMap(componentMetricMap, id); + } + boolean requestAll = getPropertyInfoMap(getComponentName(resource), id, propertyInfoMap); for (Map.Entry<String, PropertyInfo> entry : propertyInfoMap.entrySet()) { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/def7e8cf/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java index 8e883cc..5e54e42 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java @@ -58,6 +58,10 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P */ private static final Pattern FIND_ARGUMENT_METHOD_ARGUMENTS_REGEX = Pattern.compile("\".*?\"|[0-9]+"); + /** + * Supported any regex inside () + */ + private static final String FIND_REGEX_IN_METRIC_REGEX = "\\([^)]+\\)"; // ----- Constructors ------------------------------------------------------ @@ -266,4 +270,35 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P return (String) method.invoke(argValue, argList.toArray(new Object[argList.size()])); } + + /** + * Adds to the componentMetricMap a specific(not regexp) + * metric for the propertyId + * + * @param componentMetricMap + * @param propertyId + */ + protected void updateComponentMetricMap( + Map<String, PropertyInfo> componentMetricMap, String propertyId) { + + String regExpKey = getRegExpKey(propertyId); + + + if (!componentMetricMap.containsKey(propertyId) && regExpKey != null && + !regExpKey.equals(propertyId)) { + + PropertyInfo propertyInfo = componentMetricMap.get(regExpKey); + if (propertyInfo != null) { + List<String> regexGroups = getRegexGroups(regExpKey, propertyId); + String key = propertyInfo.getPropertyId(); + for (String regexGroup : regexGroups) { + regexGroup = regexGroup.replace("/", "."); + key = key.replaceFirst(FIND_REGEX_IN_METRIC_REGEX, regexGroup); + } + componentMetricMap.put(propertyId, new PropertyInfo(key, + propertyInfo.isTemporal(), propertyInfo.isPointInTime())); + } + + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/def7e8cf/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java index 3e5e8d7..d92959d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java @@ -26,11 +26,7 @@ import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -81,7 +77,7 @@ public abstract class BaseProvider { this.patterns = new HashMap<String, Pattern>(); for (String id : this.combinedIds) { if (containsArguments(id)) { - String pattern = id.replaceAll("\\$\\d+(\\.\\S+\\(\\S+\\))*", "\\\\S+"); + String pattern = id.replaceAll("\\$\\d+(\\.\\S+\\(\\S+\\))*", "(\\\\S+)"); patterns.put(id, Pattern.compile(pattern)); } } @@ -211,6 +207,29 @@ public abstract class BaseProvider { return regExpKey; } + /** + * Extracts set of matcher.group() from id + * @param regExpKey + * @param id + * @return extracted regex groups from id + */ + protected List<String> getRegexGroups(String regExpKey, String id) { + Pattern pattern = patterns.get(regExpKey); + List<String> regexGroups = new ArrayList<String>(); + + if (pattern != null) { + Matcher matcher = pattern.matcher(id); + + if (matcher.matches()) { + for (int i=0; i<matcher.groupCount(); i++){ + regexGroups.add(matcher.group(i + 1)); + } + } + } + + return regexGroups; + } + protected boolean isPatternKey(String id) { return patterns.containsKey(id); } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/def7e8cf/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java index 1fc1e02..88322f7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java @@ -318,7 +318,7 @@ public class GangliaPropertyProviderTest { Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); String expected = (configuration.isGangliaSSL() ? "https" : "http") + - "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1"; + "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&m=flume.CHANNEL.c1.ChannelCapacity&s=10&e=20&r=1"; Assert.assertEquals(expected, streamProvider.getLastSpec()); Assert.assertEquals(3, PropertyHelper.getProperties(resource).size()); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/def7e8cf/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AbstractPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AbstractPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AbstractPropertyProviderTest.java index 8a53960..02ce79e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AbstractPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AbstractPropertyProviderTest.java @@ -89,6 +89,62 @@ public class AbstractPropertyProviderTest { Assert.assertEquals("category/name1/oo_post/name2/bar_post", newPropertyId); } + @Test + public void testUpdateComponentMetricMapHDP1() { + Map<String, Map<String, PropertyInfo>> componentMetrics = + PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, + PropertyHelper.MetricsVersion.HDP1); + + AbstractPropertyProvider provider = new TestPropertyProvider(componentMetrics); + + Map<String, PropertyInfo> flumeMetrics = provider.getComponentMetrics().get( + "FLUME_SERVER"); + + int metricsBefore = flumeMetrics.size(); + String specificMetric = "metrics/flume/arg1/CHANNEL/arg2/ChannelCapacity"; + String specificPropertyInfoId = "arg1.CHANNEL.arg2.ChannelCapacity"; + Map<String, PropertyInfo> componentMetricMap = + provider.getComponentMetrics().get("FLUME_SERVER"); + + Assert.assertNull(flumeMetrics.get(specificMetric)); + + provider.updateComponentMetricMap(componentMetricMap, specificMetric); + + Assert.assertEquals(metricsBefore + 1, flumeMetrics.size()); + Assert.assertNotNull(flumeMetrics.get(specificMetric)); + Assert.assertEquals(specificPropertyInfoId, + flumeMetrics.get(specificMetric).getPropertyId()); + } + + @Test + public void testUpdateComponentMetricMapHDP2() { + Map<String, Map<String, PropertyInfo>> componentMetrics = + PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, + PropertyHelper.MetricsVersion.HDP2); + + AbstractPropertyProvider provider = new TestPropertyProvider(componentMetrics); + + Map<String, PropertyInfo> resourceManagerMetrics = provider.getComponentMetrics().get( + "RESOURCEMANAGER"); + + int metricsBefore = resourceManagerMetrics.size(); + String specificMetric = + "metrics/yarn/Queue/specificQueue1/specificQueue2/AvailableMB"; + String specificPropertyInfoId = + "yarn.QueueMetrics.Queue=specificQueue1.specificQueue2.AvailableMB"; + Map<String, PropertyInfo> componentMetricMap = + provider.getComponentMetrics().get("RESOURCEMANAGER"); + + Assert.assertNull(resourceManagerMetrics.get(specificMetric)); + + provider.updateComponentMetricMap(componentMetricMap, specificMetric); + + Assert.assertEquals(metricsBefore + 1, resourceManagerMetrics.size()); + Assert.assertNotNull(resourceManagerMetrics.get(specificMetric)); + Assert.assertEquals(specificPropertyInfoId, + resourceManagerMetrics.get(specificMetric).getPropertyId()); + } + static class TestPropertyProvider extends AbstractPropertyProvider { public TestPropertyProvider(Map<String, Map<String, PropertyInfo>> componentMetrics) { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/def7e8cf/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseProviderTest.java index e5bc59e..c8fcf70 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseProviderTest.java @@ -252,6 +252,26 @@ public class BaseProviderTest { Assert.assertEquals("value23", resource.getPropertyValue("cat4/mapMapProperty4/subMap2/key3")); } + @Test + public void testRegexpMethods() { + Set<String> propertyIds = new HashSet<String>(); + String regexp = "cat/$1.replaceAll(\\\"([.])\\\",\\\"/\\\")/key"; + String propertyId = "cat/sub/key"; + String regexp2 = "cat/$1.replaceAll(\\\"([.])\\\",\\\"/\\\")/something/$2/key"; + String propertyId2 = "cat/sub/something/sub2/key"; + + String incorrectPropertyId = "some/property/id"; + propertyIds.add(regexp); + propertyIds.add(regexp2); + + BaseProvider provider = new TestProvider(propertyIds); + Assert.assertEquals(regexp, provider.getRegExpKey(propertyId)); + Assert.assertNull(provider.getRegExpKey(incorrectPropertyId)); + Assert.assertEquals("sub", provider.getRegexGroups(regexp, propertyId).get(0)); + Assert.assertEquals("sub2", provider.getRegexGroups(regexp2, propertyId2).get(1)); + Assert.assertTrue(provider.getRegexGroups(regexp, incorrectPropertyId).isEmpty()); + } + static class TestProvider extends BaseProvider { public TestProvider(Set<String> propertyIds) {
