Repository: ambari Updated Branches: refs/heads/trunk 97eec61d1 -> 9371bb760
AMBARI-11260. Cluster aggregate metric graphs are showing averages, not sum. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9371bb76 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9371bb76 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9371bb76 Branch: refs/heads/trunk Commit: 9371bb7601c42b250ec258b85a95a5cc1f55ba61 Parents: 97eec61 Author: Siddharth Wagle <[email protected]> Authored: Thu May 21 10:08:26 2015 -0700 Committer: Siddharth Wagle <[email protected]> Committed: Thu May 21 10:08:26 2015 -0700 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 92 ++++++++++++----- .../internal/AbstractPropertyProvider.java | 4 +- .../metrics/MetricsReportPropertyProvider.java | 2 + .../MetricsReportPropertyProviderProxy.java | 15 +++ .../timeline/AMSComponentPropertyProvider.java | 58 +---------- .../metrics/timeline/AMSPropertyProvider.java | 34 ++----- .../timeline/AMSReportPropertyProvider.java | 42 +++++++- .../controller/utilities/PropertyHelper.java | 76 ++++++++++++++ .../timeline/AMSPropertyProviderTest.java | 2 + .../timeline/AMSReportPropertyProviderTest.java | 50 +++++++++- .../ams/aggregate_cluster_metrics.json | 100 +++++++++++++++++++ 11 files changed, 361 insertions(+), 114 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 0761614..f411aaa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -18,24 +18,10 @@ package org.apache.ambari.server.api.services; -import java.io.File; -import java.io.FileReader; -import java.io.FilenameFilter; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.Set; - -import javax.xml.bind.JAXBException; - +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ParentObjectNotFoundException; import org.apache.ambari.server.StackAccessException; @@ -71,16 +57,32 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptorFactory; +import org.apache.ambari.server.state.stack.Metric; import org.apache.ambari.server.state.stack.MetricDefinition; import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.state.stack.UpgradePack; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import com.google.inject.Inject; -import com.google.inject.Singleton; +import javax.xml.bind.JAXBException; +import java.io.File; +import java.io.FileReader; +import java.io.FilenameFilter; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; + +import static org.apache.ambari.server.controller.spi.Resource.InternalType.Component; +import static org.apache.ambari.server.controller.utilities.PropertyHelper.AGGREGATE_FUNCTION_IDENTIFIERS; /** @@ -867,7 +869,7 @@ public class AmbariMetaInfo { try { map = gson.fromJson(new FileReader(svc.getMetricsFile()), type); - svc.setMetrics(map); + svc.setMetrics(updateComponentMetricMapWithAggregateFunctionIds(map)); } catch (Exception e) { LOG.error ("Could not read the metrics file", e); @@ -879,6 +881,50 @@ public class AmbariMetaInfo { } /** + * Add aggregate function support for all stack defined metrics. + */ + private Map<String, Map<String, List<MetricDefinition>>> updateComponentMetricMapWithAggregateFunctionIds( + Map<String, Map<String, List<MetricDefinition>>> metricMap) { + + if (!metricMap.isEmpty()) { + // For every Component + for (Map<String, List<MetricDefinition>> componentMetricDef : metricMap.values()) { + // For every Component / HostComponent category + for (Map.Entry<String, List<MetricDefinition>> metricDefEntry : componentMetricDef.entrySet()) { + // NOTE: Only Component aggregates supported for now. + if (metricDefEntry.getKey().equals(Component.name())) { + //For every metric definition + for (MetricDefinition metricDefinition : metricDefEntry.getValue()) { + Map<String, Metric> newMetrics = new HashMap<String, Metric>(); + // Metrics System metrics only + if (metricDefinition.getType().equals("ganglia")) { + // For every function id + for (String identifierToAdd : AGGREGATE_FUNCTION_IDENTIFIERS) { + for (Map.Entry<String, Metric> metricEntry : metricDefinition.getMetrics().entrySet()) { + String newMetricKey = metricEntry.getKey() + identifierToAdd; + Metric currentMetric = metricEntry.getValue(); + Metric newMetric = new Metric( + currentMetric.getName() + identifierToAdd, + currentMetric.isPointInTime(), + currentMetric.isTemporal(), + currentMetric.isAmsHostMetric(), + currentMetric.getUnit() + ); + newMetrics.put(newMetricKey, newMetric); + } + } + } + metricDefinition.getMetrics().putAll(newMetrics); + } + } + } + } + } + + return metricMap; + } + + /** * Gets the metrics for a Role (component). * @return the list of defined metrics. */ http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java index 98b42e0..96fa24b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java @@ -21,6 +21,8 @@ package org.apache.ambari.server.controller.internal; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.text.DecimalFormat; +import java.util.Date; +import java.util.EnumMap; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -32,6 +34,7 @@ import java.util.regex.Pattern; import org.apache.ambari.server.controller.metrics.MetricReportingAdapter; import org.apache.ambari.server.controller.spi.PropertyProvider; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; /** @@ -384,5 +387,4 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P return length > 0 ? dataPoints[length - 1][0] : 0; } } - } http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java index 20e11ad..5399436 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProvider.java @@ -35,6 +35,8 @@ public abstract class MetricsReportPropertyProvider extends AbstractPropertyProv protected final ComponentSSLConfiguration configuration; + protected static final MetricsPaddingMethod DEFAULT_PADDING_METHOD = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS); // ----- Constants -------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java index 1bcf01a..a92cb37 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsReportPropertyProviderProxy.java @@ -80,6 +80,21 @@ public class MetricsReportPropertyProviderProxy extends AbstractPropertyProvider clusterNamePropertyId); } + /** + * Allow delegates to support special properties not stack defined. + */ + @Override + public Set<String> checkPropertyIds(Set<String> propertyIds) { + MetricsService metricsService = metricsServiceProvider.getMetricsServiceType(); + Set<String> checkedPropertyIds = super.checkPropertyIds(propertyIds); + + if (metricsService != null && metricsService.equals(TIMELINE_METRICS)) { + return amsMetricsReportProvider.checkPropertyIds(checkedPropertyIds); + } else { + return checkedPropertyIds; + } + } + @Override public Set<Resource> populateResources(Set<Resource> resources, Request request, Predicate predicate) throws SystemException { http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java index ded96cf..d5f415a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSComponentPropertyProvider.java @@ -20,17 +20,10 @@ package org.apache.ambari.server.controller.metrics.timeline; import org.apache.ambari.server.configuration.ComponentSSLConfiguration; import org.apache.ambari.server.controller.internal.PropertyInfo; import org.apache.ambari.server.controller.metrics.MetricHostProvider; -import org.apache.ambari.server.controller.spi.Predicate; -import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; -import org.apache.ambari.server.controller.utilities.PredicateHelper; import org.apache.ambari.server.controller.utilities.StreamProvider; -import org.apache.commons.lang.StringUtils; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; public class AMSComponentPropertyProvider extends AMSPropertyProvider { @@ -41,59 +34,10 @@ public class AMSComponentPropertyProvider extends AMSPropertyProvider { String clusterNamePropertyId, String componentNamePropertyId) { - super(updateComponentMetricsWithAggregateFunctionSupport(componentPropertyInfoMap), - streamProvider, configuration, hostProvider, + super(componentPropertyInfoMap, streamProvider, configuration, hostProvider, clusterNamePropertyId, null, componentNamePropertyId); } - /** - * This method adds supported propertyInfo for component metrics with - * aggregate function ids. API calls with multiple aggregate functions - * applied to a single metric need this support. - * - * Currently this support is added only for Component metrics, - * this can be easily extended to all levels by moving this method to the - * base class: @AMSPropertyProvider. - */ - private static Map<String, Map<String, PropertyInfo>> updateComponentMetricsWithAggregateFunctionSupport( - Map<String, Map<String, PropertyInfo>> componentMetrics) { - - if (componentMetrics == null || componentMetrics.isEmpty()) { - return componentMetrics; - } - - // For every component - for (Map<String, PropertyInfo> componentMetricInfo : componentMetrics.values()) { - Map<String, PropertyInfo> aggregateMetrics = new HashMap<String, PropertyInfo>(); - // For every metric - for (Map.Entry<String, PropertyInfo> metricEntry : componentMetricInfo.entrySet()) { - // For each aggregate function id - for (String identifierToAdd : aggregateFunctionIdentifierMap.values()) { - String metricInfoKey = metricEntry.getKey() + identifierToAdd; - // This disallows metric key suffix of the form "._sum._sum" for - // the sake of avoiding duplicates - if (componentMetricInfo.containsKey(metricInfoKey)) { - continue; - } - - PropertyInfo propertyInfo = metricEntry.getValue(); - PropertyInfo metricInfoValue = new PropertyInfo( - propertyInfo.getPropertyId() + identifierToAdd, - propertyInfo.isTemporal(), - propertyInfo.isPointInTime()); - metricInfoValue.setAmsHostMetric(propertyInfo.isAmsHostMetric()); - metricInfoValue.setAmsId(propertyInfo.getAmsId()); - metricInfoValue.setUnit(propertyInfo.getUnit()); - - aggregateMetrics.put(metricInfoKey, metricInfoValue); - } - } - componentMetricInfo.putAll(aggregateMetrics); - } - - return componentMetrics; - } - @Override protected String getHostName(Resource resource) { return null; http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java index d5af2b7..1011ded 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java @@ -39,13 +39,13 @@ import org.codehaus.jackson.map.AnnotationIntrospector; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectReader; import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.SocketTimeoutException; import java.util.Collection; import java.util.Collections; -import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; + import static org.apache.ambari.server.Role.HBASE_MASTER; import static org.apache.ambari.server.Role.HBASE_REGIONSERVER; import static org.apache.ambari.server.Role.METRICS_COLLECTOR; @@ -67,14 +68,6 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider { private final static ObjectReader timelineObjectReader; private static final String METRIC_REGEXP_PATTERN = "\\([^)]*\\)"; private static final int COLLECTOR_DEFAULT_PORT = 6188; - protected static enum AGGREGATE_FUNCTION_IDENTIFIER { - SUM, - MIN, - MAX, - AVG - } - protected static final EnumMap<AGGREGATE_FUNCTION_IDENTIFIER, String> aggregateFunctionIdentifierMap = - new EnumMap<AGGREGATE_FUNCTION_IDENTIFIER, String>(AGGREGATE_FUNCTION_IDENTIFIER.class); static { TIMELINE_APPID_MAP.put(HBASE_MASTER.name(), "HBASE"); @@ -87,11 +80,6 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider { //noinspection deprecation mapper.getSerializationConfig().setSerializationInclusion(Inclusion.NON_NULL); timelineObjectReader = mapper.reader(TimelineMetrics.class); - - aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.SUM, "._sum"); - aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.MIN, "._min"); - aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.MAX, "._max"); - aggregateFunctionIdentifierMap.put(AGGREGATE_FUNCTION_IDENTIFIER.AVG, "._avg"); } public AMSPropertyProvider(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap, @@ -121,28 +109,18 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider { */ @Override public Set<String> checkPropertyIds(Set<String> propertyIds) { + Set<String> supportedIds = new HashSet<String>(); for (String propertyId : propertyIds) { if (propertyId.startsWith(ZERO_PADDING_PARAM) - || hasAggregateFunctionSuffix(propertyId)) { - propertyIds.remove(propertyId); + || PropertyHelper.hasAggregateFunctionSuffix(propertyId)) { + supportedIds.add(propertyId); } } + propertyIds.removeAll(supportedIds); return propertyIds; } /** - * Check if property ends with a trailing suffix - */ - protected boolean hasAggregateFunctionSuffix(String propertyId) { - for (String aggregateFunctionId : aggregateFunctionIdentifierMap.values()) { - if (propertyId.endsWith(aggregateFunctionId)) { - return true; - } - } - return false; - } - - /** * The information required to make a single call to the Metrics service. */ class MetricsRequest { http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java index febcf75..2dbff68 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.controller.metrics.timeline; import org.apache.ambari.server.configuration.ComponentSSLConfiguration; import org.apache.ambari.server.controller.internal.PropertyInfo; import org.apache.ambari.server.controller.metrics.MetricHostProvider; +import org.apache.ambari.server.controller.metrics.MetricsPaddingMethod; import org.apache.ambari.server.controller.metrics.MetricsPropertyProvider; import org.apache.ambari.server.controller.metrics.MetricsReportPropertyProvider; import org.apache.ambari.server.controller.spi.Predicate; @@ -27,6 +28,7 @@ import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.TemporalInfo; +import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.apache.ambari.server.controller.utilities.StreamProvider; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; @@ -41,16 +43,21 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.SocketTimeoutException; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.ZERO_PADDING_PARAM; import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService.TIMELINE_METRICS; +import static org.apache.ambari.server.controller.utilities.PropertyHelper.updateMetricsWithAggregateFunctionSupport; public class AMSReportPropertyProvider extends MetricsReportPropertyProvider { private static ObjectMapper mapper; private final static ObjectReader timelineObjectReader; + private MetricsPaddingMethod metricsPaddingMethod; static { mapper = new ObjectMapper(); @@ -71,6 +78,22 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider { hostProvider, clusterNamePropertyId); } + /** + * Support properties with aggregate functions and metrics padding method. + */ + @Override + public Set<String> checkPropertyIds(Set<String> propertyIds) { + Set<String> supportedIds = new HashSet<String>(); + for (String propertyId : propertyIds) { + if (propertyId.startsWith(ZERO_PADDING_PARAM) + || PropertyHelper.hasAggregateFunctionSuffix(propertyId)) { + supportedIds.add(propertyId); + } + } + propertyIds.removeAll(supportedIds); + return propertyIds; + } + @Override public Set<Resource> populateResources(Set<Resource> resources, Request request, Predicate predicate) throws SystemException { @@ -96,13 +119,27 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider { * @throws SystemException if unable to populate the resource */ private boolean populateResource(Resource resource, Request request, Predicate predicate) - throws SystemException { + throws SystemException { Set<String> propertyIds = getPropertyIds(); if (propertyIds.isEmpty()) { return true; } + + metricsPaddingMethod = DEFAULT_PADDING_METHOD; + + Set<String> requestPropertyIds = request.getPropertyIds(); + if (requestPropertyIds != null && !requestPropertyIds.isEmpty()) { + for (String propertyId : requestPropertyIds) { + if (propertyId.startsWith(ZERO_PADDING_PARAM)) { + String paddingStrategyStr = propertyId.substring(ZERO_PADDING_PARAM.length() + 1); + metricsPaddingMethod = new MetricsPaddingMethod( + MetricsPaddingMethod.PADDING_STRATEGY.valueOf(paddingStrategyStr)); + } + } + } + String clusterName = (String) resource.getPropertyValue(clusterNamePropertyId); // Check liveliness of host @@ -165,6 +202,9 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider { for (TimelineMetric metric : timelineMetrics.getMetrics()) { if (metric.getMetricName() != null && metric.getMetricValues() != null) { + // Pad zeros or nulls if needed + metricsPaddingMethod.applyPaddingStrategy(metric, temporalInfo); + String propertyId = propertyIdMap.get(metric.getMetricName()); if (propertyId != null) { resource.setProperty(propertyId, getValue(metric, true)); http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java index cde8581..cefe953 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java @@ -20,8 +20,10 @@ package org.apache.ambari.server.controller.utilities; import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; @@ -34,6 +36,7 @@ import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SortRequest; import org.apache.ambari.server.controller.spi.TemporalInfo; +import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -49,6 +52,15 @@ public class PropertyHelper { private static final String KEY_PROPERTIES_FILE = "key_properties.json"; private static final char EXTERNAL_PATH_SEP = '/'; + /** + * Aggregate functions implicitly supported by the Metrics Service + */ + public static final List<String> AGGREGATE_FUNCTION_IDENTIFIERS = + Arrays.asList("._sum", "._max", "._min", "._avg"); + + private static final List<Resource.InternalType> REPORT_METRIC_RESOURCES = + Arrays.asList(Resource.InternalType.Cluster, Resource.InternalType.Host); + private static final Map<Resource.InternalType, Set<String>> PROPERTY_IDS = readPropertyIds(PROPERTIES_FILE); private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> JMX_PROPERTY_IDS = readPropertyProviderIds(JMX_PROPERTIES_FILE); private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> GANGLIA_PROPERTY_IDS = readPropertyProviderIds(GANGLIA_PROPERTIES_FILE); @@ -439,6 +451,9 @@ public class PropertyHelper { } componentMetrics.put(componentEntry.getKey(), metrics); } + if (REPORT_METRIC_RESOURCES.contains(resourceEntry.getKey())) { + updateMetricsWithAggregateFunctionSupport(componentMetrics); + } resourceMetrics.put(resourceEntry.getKey(), componentMetrics); } return resourceMetrics; @@ -550,4 +565,65 @@ public class PropertyHelper { return unit; } } + + /** + * This method adds supported propertyInfo for component metrics with + * aggregate function ids. API calls with multiple aggregate functions + * applied to a single metric need this support. + * + * Currently this support is added only for Component & Report metrics. + * This can be easily extended by making the call from the appropriate + * sub class of: @AMSPropertyProvider. + * + */ + public static void updateMetricsWithAggregateFunctionSupport(Map<String, Map<String, PropertyInfo>> metrics) { + + if (metrics == null || metrics.isEmpty()) { + return; + } + + // For every component + for (Map.Entry<String, Map<String, PropertyInfo>> metricInfoEntry : metrics.entrySet()) { + Map<String, PropertyInfo> metricInfo = metricInfoEntry.getValue(); + + Map<String, PropertyInfo> aggregateMetrics = new HashMap<String, PropertyInfo>(); + // For every metric + for (Map.Entry<String, PropertyInfo> metricEntry : metricInfo.entrySet()) { + // For each aggregate function id + for (String identifierToAdd : AGGREGATE_FUNCTION_IDENTIFIERS) { + String metricInfoKey = metricEntry.getKey() + identifierToAdd; + // This disallows metric key suffix of the form "._sum._sum" for + // the sake of avoiding duplicates + if (metricInfo.containsKey(metricInfoKey)) { + continue; + } + + PropertyInfo propertyInfo = metricEntry.getValue(); + PropertyInfo metricInfoValue = new PropertyInfo( + propertyInfo.getPropertyId() + identifierToAdd, + propertyInfo.isTemporal(), + propertyInfo.isPointInTime()); + metricInfoValue.setAmsHostMetric(propertyInfo.isAmsHostMetric()); + metricInfoValue.setAmsId(!StringUtils.isEmpty(propertyInfo.getAmsId()) ? + propertyInfo.getAmsId() + identifierToAdd : null); + metricInfoValue.setUnit(propertyInfo.getUnit()); + + aggregateMetrics.put(metricInfoKey, metricInfoValue); + } + } + metricInfo.putAll(aggregateMetrics); + } + } + + /** + * Check if property ends with a trailing suffix of function id. + */ + public static boolean hasAggregateFunctionSuffix(String propertyId) { + for (String aggregateFunctionId : AGGREGATE_FUNCTION_IDENTIFIERS) { + if (propertyId.endsWith(aggregateFunctionId)) { + return true; + } + } + return false; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java index 81c53cc..923f2f7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java @@ -47,6 +47,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -444,6 +445,7 @@ public class AMSPropertyProviderTest { Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Component); + PropertyHelper.updateMetricsWithAggregateFunctionSupport(propertyIds); AMSComponentPropertyProvider propertyProvider = new AMSComponentPropertyProvider( propertyIds, http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java index 8fba0e7..c0ce419 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProviderTest.java @@ -42,6 +42,7 @@ public class AMSReportPropertyProviderTest { private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Clusters", "cluster_name"); private static final String FILE_PATH_PREFIX = "ams" + File.separator; private static final String SINGLE_HOST_METRICS_FILE_PATH = FILE_PATH_PREFIX + "single_host_metric.json"; + private static final String AGGREGATE_CLUSTER_METRICS_FILE_PATH = FILE_PATH_PREFIX + "aggregate_cluster_metrics.json"; @Test public void testPopulateResources() throws Exception { @@ -59,13 +60,13 @@ public class AMSReportPropertyProviderTest { sslConfiguration, metricHostProvider, CLUSTER_NAME_PROPERTY_ID - ); + ); String propertyId = PropertyHelper.getPropertyId("metrics/cpu", "User"); Resource resource = new ResourceImpl(Resource.Type.Cluster); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); - temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416528819369L, 1416528819569L, 1L)); + temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416445244701L, 1416445244901L, 1L)); Request request = PropertyHelper.getReadRequest( Collections.singleton(propertyId), temporalInfoMap); Set<Resource> resources = @@ -77,10 +78,51 @@ public class AMSReportPropertyProviderTest { URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 8188); uriBuilder.addParameter("metricNames", "cpu_user"); uriBuilder.addParameter("appId", "HOST"); - uriBuilder.addParameter("startTime", "1416528819369"); - uriBuilder.addParameter("endTime", "1416528819569"); + uriBuilder.addParameter("startTime", "1416445244701"); + uriBuilder.addParameter("endTime", "1416445244901"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Number[][] val = (Number[][]) res.getPropertyValue("metrics/cpu/User"); Assert.assertEquals(111, val.length); } + + @Test + public void testPopulateResourceWithAggregateFunction() throws Exception { + TestStreamProvider streamProvider = new TestStreamProvider(AGGREGATE_CLUSTER_METRICS_FILE_PATH); + TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); + ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); + + Map<String, Map<String, PropertyInfo>> propertyIds = + PropertyHelper.getMetricPropertyIds(Resource.Type.Cluster); + + AMSReportPropertyProvider propertyProvider = + new AMSReportPropertyProvider( + propertyIds, + streamProvider, + sslConfiguration, + metricHostProvider, + CLUSTER_NAME_PROPERTY_ID + ); + + String propertyId = PropertyHelper.getPropertyId("metrics/cpu", "User._sum"); + Resource resource = new ResourceImpl(Resource.Type.Cluster); + resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); + Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); + temporalInfoMap.put(propertyId, new TemporalInfoImpl(1432033256912L, 1432033257912L, 1L)); + Request request = PropertyHelper.getReadRequest( + Collections.singleton(propertyId), temporalInfoMap); + Set<Resource> resources = + propertyProvider.populateResources(Collections.singleton(resource), request, null); + Assert.assertEquals(1, resources.size()); + Resource res = resources.iterator().next(); + Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); + Assert.assertNotNull(properties); + URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 8188); + uriBuilder.addParameter("metricNames", "cpu_user._sum"); + uriBuilder.addParameter("appId", "HOST"); + uriBuilder.addParameter("startTime", "1432033256912"); + uriBuilder.addParameter("endTime", "1432033257912"); + Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); + Number[][] val = (Number[][]) res.getPropertyValue("metrics/cpu/User._sum"); + Assert.assertEquals(91, val.length); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9371bb76/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json b/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json new file mode 100644 index 0000000..d3987cf --- /dev/null +++ b/ambari-server/src/test/resources/ams/aggregate_cluster_metrics.json @@ -0,0 +1,100 @@ +{"metrics": [ + { + "timestamp": 1432033257912, + "metricname": "cpu_user._sum", + "appid": "HOST", + "starttime": 1432033257912, + "metrics": { + "1432033257912": 10347.5, + "1432033287912": 11821.25, + "1432033317912": 10200.0, + "1432033347912": 12147.5, + "1432033377912": 10530.3125, + "1432033407912": 12243.75, + "1432033437912": 10220.0, + "1432033467912": 12512.5, + "1432033497912": 10302.5, + "1432033527912": 12165.625, + "1432033557912": 10200.0, + "1432033587912": 12478.75, + "1432033617912": 10299.6875, + "1432033647912": 12532.5, + "1432033677912": 10180.0, + "1432033707912": 12047.5, + "1432033737912": 10340.9375, + "1432033767912": 11628.75, + "1432033797912": 10140.0, + "1432033827912": 12151.25, + "1432033857912": 10375.3125, + "1432033887912": 11683.125, + "1432033917912": 10200.0, + "1432033947912": 12177.5, + "1432033977912": 10372.8125, + "1432034007912": 11806.25, + "1432034037912": 10180.0, + "1432034067912": 11233.75, + "1432034097912": 10389.0625, + "1432034127912": 11730.0, + "1432034157912": 10240.0, + "1432034187912": 11493.125, + "1432034217912": 10360.0, + "1432034247912": 11346.25, + "1432034277912": 10240.0, + "1432034307912": 11329.375, + "1432034337912": 10391.5625, + "1432034367912": 11148.125, + "1432034397912": 10240.0, + "1432034427912": 11193.125, + "1432034457912": 10399.6875, + "1432034487912": 11188.125, + "1432034517912": 10280.0, + "1432034547912": 11183.125, + "1432034577912": 10455.625, + "1432034607912": 12975.625, + "1432034637912": 10280.0, + "1432034667912": 13041.25, + "1432034697912": 10297.5, + "1432034727912": 13141.875, + "1432034757912": 10260.0, + "1432034787912": 12880.0, + "1432034817912": 10288.75, + "1432034847912": 13072.5, + "1432034877912": 10240.0, + "1432034907912": 13304.375, + "1432034937912": 10306.25, + "1432034967912": 13011.25, + "1432034997912": 10270.0, + "1432035027912": 12463.75, + "1432035057912": 10339.6875, + "1432035087912": 12656.875, + "1432035117912": 10270.0, + "1432035147912": 12428.125, + "1432035177912": 10347.8125, + "1432035207912": 12195.0, + "1432035237912": 10320.0, + "1432035267912": 11796.25, + "1432035297912": 10418.4375, + "1432035327912": 11716.25, + "1432035357912": 10390.0, + "1432035387912": 11400.0, + "1432035417912": 10463.125, + "1432035447912": 11177.5, + "1432035477912": 10390.0, + "1432035507912": 11073.125, + "1432035537912": 10460.9375, + "1432035567912": 10285.0, + "1432035597912": 10350.0, + "1432035627912": 10968.75, + "1432035657912": 10440.3125, + "1432035687912": 10471.25, + "1432035717912": 10380.0, + "1432035747912": 10379.375, + "1432035777912": 10483.4375, + "1432035807912": 10336.875, + "1432035837912": 10290.0, + "1432035867912": 10173.75, + "1432035897912": 10477.8125, + "1432035927912": 10346.875 + } + } +]} \ No newline at end of file
