Repository: ambari Updated Branches: refs/heads/trunk 01ff6a79b -> 7270881fb
AMBARI-7398. Add metrics support to Slider View Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7270881f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7270881f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7270881f Branch: refs/heads/trunk Commit: 7270881fbe18f1b115c2551ff86db290ddd406c2 Parents: 01ff6a7 Author: Sumit Mohanty <[email protected]> Authored: Thu Sep 18 18:59:09 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Thu Sep 18 18:59:09 2014 -0700 ---------------------------------------------------------------------- contrib/views/slider/pom.xml | 9 +- .../ambari/view/slider/GangliaMetric.java | 257 +++++++++++++++++++ .../ambari/view/slider/MetricsHolder.java | 44 ++++ .../apache/ambari/view/slider/SliderApp.java | 16 +- .../ambari/view/slider/SliderAppType.java | 32 +-- .../slider/SliderAppsViewControllerImpl.java | 87 +++++-- .../apache/ambari/view/slider/TemporalInfo.java | 48 ++++ .../rest/client/SliderAppGangliaHelper.java | 196 ++++++++++++++ .../rest/client/SliderAppMasterClient.java | 127 +++++++-- 9 files changed, 740 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/views/slider/pom.xml b/contrib/views/slider/pom.xml index 8c33c93..49348df 100644 --- a/contrib/views/slider/pom.xml +++ b/contrib/views/slider/pom.xml @@ -331,6 +331,13 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${httpclient.version}</version> + <exclusions> + </exclusions> + </dependency> + <dependency> <groupId>com.beust</groupId> <artifactId>jcommander</artifactId> <version>${jcommander.version}</version> @@ -397,7 +404,7 @@ <bigtop.version>0.7.0</bigtop.version> <commons-configuration.version>1.6</commons-configuration.version> <commons-lang.version>2.6</commons-lang.version> - <httpclient.version>3.1</httpclient.version> + <httpclient.version>4.2.5</httpclient.version> <jcommander.version>1.30</jcommander.version> <curator.version>2.4.1</curator.version> <zookeeper.version>3.4.5</zookeeper.version> http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java new file mode 100644 index 0000000..cb51768 --- /dev/null +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/GangliaMetric.java @@ -0,0 +1,257 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.slider; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +@JsonIgnoreProperties(ignoreUnknown = true) + + +/** + * Data structure for temporal data returned from Ganglia Web. + */ +public class GangliaMetric { + + // Note that the member names correspond to the names in the JSON returned from Ganglia Web. + + /** + * The name. + */ + private String ds_name; + + /** + * The ganglia cluster name. + */ + private String cluster_name; + + /** + * The graph type. + */ + private String graph_type; + + /** + * The host name. + */ + private String host_name; + + /** + * The metric name. + */ + private String metric_name; + + /** + * The temporal data points. + */ + private Number[][] datapoints; + + + private static final Set<String> PERCENTAGE_METRIC; + + //BUG-3386 Cluster CPU Chart is off the charts + // Here can be added other percentage metrics + static { + Set<String> temp = new HashSet<String>(); + temp.add("cpu_wio"); + temp.add("cpu_idle"); + temp.add("cpu_nice"); + temp.add("cpu_aidle"); + temp.add("cpu_system"); + temp.add("cpu_user"); + PERCENTAGE_METRIC = Collections.unmodifiableSet(temp); + } + + + // ----- GangliaMetric ----------------------------------------------------- + + public String getDs_name() { + return ds_name; + } + + public void setDs_name(String ds_name) { + this.ds_name = ds_name; + } + + public String getCluster_name() { + return cluster_name; + } + + public void setCluster_name(String cluster_name) { + this.cluster_name = cluster_name; + } + + public String getGraph_type() { + return graph_type; + } + + public void setGraph_type(String graph_type) { + this.graph_type = graph_type; + } + + public String getHost_name() { + return host_name; + } + + public void setHost_name(String host_name) { + this.host_name = host_name; + } + + public String getMetric_name() { + return metric_name; + } + + public void setMetric_name(String metric_name) { + this.metric_name = metric_name; + } + + public Number[][] getDatapoints() { + return datapoints; + } + + + public void setDatapoints(Number[][] datapoints) { + this.datapoints = datapoints; + } + + public void setDatapointsFromList(List<GangliaMetric.TemporalMetric> listTemporalMetrics) { + //this.datapoints = datapoints; + Number[][] datapointsArray = new Number[listTemporalMetrics.size()][2]; + int cnt = 0; + if (PERCENTAGE_METRIC.contains(metric_name)) { + int firstIndex = 0; + int lastIndex = listTemporalMetrics.size() - 1; + for (int i = firstIndex; i <= lastIndex; ++i) { + GangliaMetric.TemporalMetric m = listTemporalMetrics.get(i); + Number val = m.getValue(); + if (100.0 >= val.doubleValue()) { + datapointsArray[cnt][0] = val; + datapointsArray[cnt][1] = m.getTime(); + cnt++; + } + } + } else { + int firstIndex = 0; + int lastIndex = listTemporalMetrics.size() - 1; + for (int i = firstIndex; i <= lastIndex; ++i) { + GangliaMetric.TemporalMetric m = listTemporalMetrics.get(i); + datapointsArray[i][0] = m.getValue(); + datapointsArray[i][1] = m.getTime(); + cnt++; + } + } + + this.datapoints = new Number[cnt][2]; + for (int i = 0; i < this.datapoints.length; i++) { + this.datapoints[i][0] = datapointsArray[i][0]; + this.datapoints[i][1] = datapointsArray[i][1]; + } + + } + + // ----- Object overrides -------------------------------------------------- + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + + stringBuilder.append("\n"); + stringBuilder.append("name="); + stringBuilder.append(ds_name); + stringBuilder.append("\n"); + stringBuilder.append("cluster name="); + stringBuilder.append(cluster_name); + stringBuilder.append("\n"); + stringBuilder.append("graph type="); + stringBuilder.append(graph_type); + stringBuilder.append("\n"); + stringBuilder.append("host name="); + stringBuilder.append(host_name); + stringBuilder.append("\n"); + stringBuilder.append("api name="); + stringBuilder.append(metric_name); + stringBuilder.append("\n"); + + stringBuilder.append("datapoints (value/timestamp):"); + stringBuilder.append("\n"); + + + boolean first = true; + stringBuilder.append("["); + for (Number[] m : datapoints) { + if (!first) { + stringBuilder.append(","); + } + stringBuilder.append("["); + stringBuilder.append(m[0]); + stringBuilder.append(","); + stringBuilder.append(m[1].longValue()); + stringBuilder.append("]"); + first = false; + } + stringBuilder.append("]"); + + return stringBuilder.toString(); + } + + public static class TemporalMetric { + private Number m_value; + private Number m_time; + private boolean valid; + + public boolean isValid() { + return valid; + } + + public TemporalMetric(String value, Number time) { + valid = true; + try{ + m_value = convertToNumber(value); + } catch (NumberFormatException e) { + valid = false; + } + m_time = time; + } + + public Number getValue() { + return m_value; + } + + public Number getTime() { + return m_time; + } + + private Number convertToNumber(String s) throws NumberFormatException { + Number res; + if(s.contains(".")){ + Double d = Double.parseDouble(s); + if(d.isNaN() || d.isInfinite()){ + throw new NumberFormatException(s); + } else { + res = d; + } + } else { + res = Long.parseLong(s); + } + return res; + } + + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java new file mode 100644 index 0000000..29f35b0 --- /dev/null +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/MetricsHolder.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.slider; + +import org.apache.ambari.view.slider.rest.client.Metric; + +import java.util.Map; + +public class MetricsHolder { + private Map<String, Map<String, Map<String, Metric>>> jmxMetrics; + private Map<String, Map<String, Map<String, Metric>>> gangliaMetrics; + + public Map<String, Map<String, Map<String, Metric>>> getJmxMetrics() { + return jmxMetrics; + } + + public void setJmxMetrics(Map<String, Map<String, Map<String, Metric>>> jmxMetrics) { + this.jmxMetrics = jmxMetrics; + } + + public Map<String, Map<String, Map<String, Metric>>> getGangliaMetrics() { + return gangliaMetrics; + } + + public void setGangliaMetrics(Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) { + this.gangliaMetrics = gangliaMetrics; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java index 6276299..d75762e 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java @@ -18,6 +18,7 @@ package org.apache.ambari.view.slider; +import java.util.List; import java.util.Map; public class SliderApp { @@ -38,8 +39,9 @@ public class SliderApp { private Map<String, String> urls; private Map<String, Map<String, String>> configs; private Map<String, SliderAppComponent> components; - private Map<String, Object> metrics; + private Map<String, Number[][]> metrics; private Map<String, Object> alerts; + private List<String> supportedMetrics; public String getName() { return name; @@ -129,14 +131,22 @@ public class SliderApp { this.components = components; } - public Map<String, Object> getMetrics() { + public Map<String, Number[][]> getMetrics() { return metrics; } - public void setMetrics(Map<String, Object> metrics) { + public void setMetrics(Map<String, Number[][]> metrics) { this.metrics = metrics; } + public List<String> getSupportedMetrics() { + return supportedMetrics; + } + + public void setSupportedMetrics(List<String> supportedMetrics) { + this.supportedMetrics = supportedMetrics; + } + public long getStartTime() { return startTime; } http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java index 528c1fe..82386af 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppType.java @@ -18,14 +18,9 @@ package org.apache.ambari.view.slider; -import org.apache.ambari.view.slider.rest.client.Metric; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; - import java.util.List; import java.util.Map; -@JsonIgnoreProperties({ "jmxMetrics", "gangliaMetrics" }) public class SliderAppType { private String id; private String typeName; @@ -34,27 +29,14 @@ public class SliderAppType { private Map<String, String> typeConfigs; private List<SliderAppTypeComponent> typeComponents; private String typePackageFileName; - @JsonIgnore - private Map<String, Map<String, Map<String, Metric>>> jmxMetrics; - @JsonIgnore - private Map<String, Map<String, Map<String, Metric>>> gangliaMetrics; - - @JsonIgnore - public Map<String, Map<String, Map<String, Metric>>> getJmxMetrics() { - return jmxMetrics; - } + private List<String> supportedMetrics; - public void setJmxMetrics(Map<String, Map<String, Map<String, Metric>>> jmxMetrics) { - this.jmxMetrics = jmxMetrics; + public List<String> getSupportedMetrics() { + return supportedMetrics; } - @JsonIgnore - public Map<String, Map<String, Map<String, Metric>>> getGangliaMetrics() { - return gangliaMetrics; - } - - public void setGangliaMetrics(Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) { - this.gangliaMetrics = gangliaMetrics; + public void setSupportedMetrics(List<String> supportedMetrics) { + this.supportedMetrics = supportedMetrics; } public String getId() { @@ -113,4 +95,8 @@ public class SliderAppType { this.typePackageFileName = typePackageFileName; } + public String uniqueName() { + return getTypeName() + "-" + getTypeVersion(); + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java index ef11cfa..6580d90 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java @@ -85,12 +85,14 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { private static final Logger logger = Logger .getLogger(SliderAppsViewControllerImpl.class); + private static String METRICS_PREFIX = "metrics/"; @Inject private ViewContext viewContext; private List<SliderAppType> appTypes; private Integer createAppCounter = -1; @Inject private SliderAppsAlerts sliderAlerts; + private Map<String, MetricsHolder> appMetrics = new HashMap<String, MetricsHolder>(); private String getAppsFolderPath() { return viewContext.getAmbariProperty("resources.dir") + "/apps"; @@ -221,10 +223,25 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } } if (properties != null && !properties.isEmpty()) { + SliderAppType matchedAppType = null; + List<SliderAppType> matchingAppTypes = getSliderAppTypes(null); + if (matchingAppTypes != null && matchingAppTypes.size() > 0) { + for (SliderAppType appType : matchingAppTypes) { + if ((appType.getTypeName() != null && appType.getTypeName() + .equalsIgnoreCase(app.getType())) + && (appType.getTypeVersion() != null && appType.getTypeVersion() + .equalsIgnoreCase(app.getAppVersion()))) { + matchedAppType = appType; + break; + } + } + } + SliderAppMasterClient sliderAppClient = yarnApp.getTrackingUrl() == null ? null : new SliderAppMasterClient(yarnApp.getTrackingUrl()); SliderAppMasterData appMasterData = null; Map<String, String> quickLinks = new HashMap<String, String>(); + Set<String> gangliaMetrics = new HashSet<String>(); for (String property : properties) { if ("RUNNING".equals(app.getState())) { if (sliderAppClient != null) { @@ -248,25 +265,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } if (quickLinks != null && quickLinks.containsKey("JMX")) { String jmxUrl = quickLinks.get("JMX"); - List<SliderAppType> appTypes = getSliderAppTypes(null); - if (appTypes != null && appTypes.size() > 0) { - for (SliderAppType appType : appTypes) { - if (logger.isDebugEnabled()) { - logger.debug("TYPE: " + appType.getTypeName() + " " - + app.getType()); - logger.debug("VERSION: " + appType.getTypeVersion() + " " - + app.getAppVersion()); - } - if ((appType.getTypeName() != null && appType.getTypeName() - .equalsIgnoreCase(app.getType())) - && (appType.getTypeVersion() != null && appType - .getTypeVersion().equalsIgnoreCase( - app.getAppVersion()))) { - app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, - appType)); - break; - } - } + if (matchedAppType != null) { + MetricsHolder metricsHolder = appMetrics.get(matchedAppType + .uniqueName()); + app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, + matchedAppType, metricsHolder)); } } Map<String, Map<String, String>> configs = sliderAppClient @@ -350,10 +353,29 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { + yarnApp.getName(), e); throw new RuntimeException(e.getMessage(), e); } + } else if (property.startsWith(METRICS_PREFIX)) { + gangliaMetrics.add(property.substring(METRICS_PREFIX.length())); + } else if ("supportedMetrics".equals(property)) { + if (matchedAppType != null) { + app.setSupportedMetrics(matchedAppType.getSupportedMetrics()); + } } } } } + if (gangliaMetrics.size() > 0) { + if (quickLinks.isEmpty()) { + quickLinks = sliderAppClient + .getQuickLinks(appMasterData.publisherUrl); + } + if (quickLinks != null && quickLinks.containsKey("Metrics")) { + String metricsUrl = quickLinks.get("Metrics"); + MetricsHolder metricsHolder = appMetrics.get(matchedAppType + .uniqueName()); + app.setMetrics(sliderAppClient.getGangliaMetrics(metricsUrl, + gangliaMetrics, null, viewContext, matchedAppType, metricsHolder)); + } + } } return app; } @@ -626,9 +648,14 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { appTypeComponentList.add(appTypeComponent); } - appType.setJmxMetrics(readMetrics(zipFile, "jmx_metrics.json")); - appType.setGangliaMetrics(readMetrics(zipFile, + MetricsHolder metricsHolder = new MetricsHolder(); + metricsHolder.setJmxMetrics(readMetrics(zipFile, + "jmx_metrics.json")); + metricsHolder.setGangliaMetrics(readMetrics(zipFile, "ganglia_metrics.json")); + appType.setSupportedMetrics(getSupportedMetrics(metricsHolder + .getGangliaMetrics())); + appMetrics.put(appType.uniqueName(), metricsHolder); appType.setTypeComponents(appTypeComponentList); appTypes.add(appType); @@ -644,19 +671,33 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { return appTypes; } + private List<String> getSupportedMetrics( + Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) { + Set<String> supportedMetrics = new HashSet<String>(); + if (gangliaMetrics != null && gangliaMetrics.size() > 0) { + for (Map<String, Map<String, Metric>> compMetrics : gangliaMetrics + .values()) { + for (Map<String, Metric> metrics : compMetrics.values()) { + supportedMetrics.addAll(metrics.keySet()); + } + } + } + return new ArrayList<String>(supportedMetrics); + } + Map<String, Map<String, Map<String, Metric>>> readMetrics(ZipFile zipFile, String fileName) { Map<String, Map<String, Map<String, Metric>>> metrics = null; try { InputStream inputStream = zipFile.getInputStream(zipFile - .getEntry("jmx_metrics.json")); + .getEntry(fileName)); ObjectMapper mapper = new ObjectMapper(); metrics = mapper.readValue(inputStream, new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() { }); } catch (IOException e) { - logger.info("Error reading metrics. " + e.getMessage()); + logger.info("Error reading metrics for file " + fileName + ". " + e.getMessage()); } return metrics; http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java new file mode 100644 index 0000000..a501615 --- /dev/null +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/TemporalInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.slider; + +/** + * Temporal query data. + */ +public interface TemporalInfo { + /** + * Get the start of the requested time range. The time is given in + * seconds since the Unix epoch. + * + * @return the start time in seconds + */ + Long getStartTime(); + + /** + * Get the end of the requested time range. The time is given in + * seconds since the Unix epoch. + * + * @return the end time in seconds + */ + Long getEndTime(); + + /** + * Get the requested time between each data point of the temporal + * data. The time is given in seconds. + * + * @return the step time in seconds + */ + Long getStep(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java new file mode 100644 index 0000000..fba7d23 --- /dev/null +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.slider.rest.client; + +import org.apache.ambari.view.SystemException; +import org.apache.ambari.view.ViewContext; +import org.apache.ambari.view.slider.GangliaMetric; +import org.apache.ambari.view.slider.TemporalInfo; +import org.apache.log4j.Logger; +import org.apache.http.client.utils.URIBuilder; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class SliderAppGangliaHelper { + + private static final Logger logger = Logger + .getLogger(SliderAppGangliaHelper.class); + + private static String getSetString(Set<String> set, int limit) { + StringBuilder sb = new StringBuilder(); + + if (limit == -1 || set.size() <= limit) { + for (String cluster : set) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(cluster); + } + } + return sb.toString(); + } + + private static Number convertToNumber(String s) { + return s.contains(".") ? Double.parseDouble(s) : Long.parseLong(s); + } + + public static Map<String, GangliaMetric> getGangliaMetrics(ViewContext context, + String spec, + String params) throws IOException { + Map<String, GangliaMetric> receivedMetrics = new HashMap<String, GangliaMetric>(); + Map<String, String> headers = new HashMap<String, String>(); + BufferedReader reader = new BufferedReader(new InputStreamReader( + context.getURLStreamProvider().readFrom(spec, "POST", params, headers))); + + String feedStart = reader.readLine(); + if (feedStart == null || feedStart.isEmpty()) { + logger.info("Empty feed while getting ganglia metrics for spec => " + + spec); + return null; + } + int startTime = convertToNumber(feedStart).intValue(); + + String dsName = reader.readLine(); + if (dsName == null || dsName.isEmpty()) { + logger.info("Feed without body while reading ganglia metrics for spec " + + "=> " + spec); + return null; + } + + while (!"[~EOF]".equals(dsName)) { + GangliaMetric metric = new GangliaMetric(); + List<GangliaMetric.TemporalMetric> listTemporalMetrics = + new ArrayList<GangliaMetric.TemporalMetric>(); + + metric.setDs_name(dsName); + metric.setCluster_name(reader.readLine()); + metric.setHost_name(reader.readLine()); + metric.setMetric_name(reader.readLine()); + + String timeStr = reader.readLine(); + String stepStr = reader.readLine(); + if (timeStr == null || timeStr.isEmpty() || stepStr == null + || stepStr.isEmpty()) { + logger.info("Unexpected end of stream reached while getting ganglia " + + "metrics for spec => " + spec); + return null; + } + int time = convertToNumber(timeStr).intValue(); + int step = convertToNumber(stepStr).intValue(); + + String val = reader.readLine(); + String lastVal = null; + + while (val != null && !"[~EOM]".equals(val)) { + if (val.startsWith("[~r]")) { + Integer repeat = Integer.valueOf(val.substring(4)) - 1; + for (int i = 0; i < repeat; ++i) { + if (!"[~n]".equals(lastVal)) { + GangliaMetric.TemporalMetric tm = new GangliaMetric.TemporalMetric(lastVal, time); + if (tm.isValid()) listTemporalMetrics.add(tm); + } + time += step; + } + } else { + if (!"[~n]".equals(val)) { + GangliaMetric.TemporalMetric tm = new GangliaMetric.TemporalMetric(val, time); + if (tm.isValid()) listTemporalMetrics.add(tm); + } + time += step; + } + lastVal = val; + val = reader.readLine(); + } + + metric.setDatapointsFromList(listTemporalMetrics); + receivedMetrics.put(metric.getMetric_name(), metric); + + dsName = reader.readLine(); + if (dsName == null || dsName.isEmpty()) { + logger.info("Unexpected end of stream reached while getting ganglia " + + "metrics for spec => " + spec); + return null; + } + } + String feedEnd = reader.readLine(); + if (feedEnd == null || feedEnd.isEmpty()) { + logger.info("Error reading end of feed while getting ganglia metrics " + + "for spec => " + spec); + } else { + + int endTime = convertToNumber(feedEnd).intValue(); + int totalTime = endTime - startTime; + if (logger.isInfoEnabled() && totalTime > 3) { + logger.info("Ganglia resource population time: " + totalTime); + } + } + return receivedMetrics; + } + + public static String getSpec(String gangliaUrl, + Set<String> metricSet, + TemporalInfo temporalInfo) throws SystemException, URISyntaxException { + + String metrics = getSetString(metricSet, -1); + + URIBuilder uriBuilder = new URIBuilder(gangliaUrl); + + uriBuilder.setParameter("h", "__SummaryInfo__"); + + if (metrics.length() > 0) { + uriBuilder.setParameter("m", metrics); + } else { + // get all metrics + uriBuilder.setParameter("m", ".*"); + } + + if (temporalInfo != null) { + long startTime = temporalInfo.getStartTime(); + if (startTime != -1) { + uriBuilder.setParameter("s", String.valueOf(startTime)); + } + + long endTime = temporalInfo.getEndTime(); + if (endTime != -1) { + uriBuilder.setParameter("e", String.valueOf(endTime)); + } + + long step = temporalInfo.getStep(); + if (step != -1) { + uriBuilder.setParameter("r", String.valueOf(step)); + } + } else { + long endTime = System.currentTimeMillis() / 1000; + long startTime = System.currentTimeMillis() / 1000 - 60 * 60; + uriBuilder.setParameter("e", String.valueOf(endTime)); + uriBuilder.setParameter("s", String.valueOf(startTime)); + uriBuilder.setParameter("r", "15"); + } + + return uriBuilder.toString(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7270881f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java index ab6289c..8bc7400 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java @@ -22,14 +22,19 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.apache.ambari.view.URLStreamProvider; import org.apache.ambari.view.ViewContext; +import org.apache.ambari.view.slider.GangliaMetric; +import org.apache.ambari.view.slider.MetricsHolder; import org.apache.ambari.view.slider.SliderAppType; import org.apache.ambari.view.slider.SliderAppTypeComponent; +import org.apache.ambari.view.slider.TemporalInfo; import org.apache.commons.httpclient.HttpException; import org.apache.log4j.Logger; @@ -94,7 +99,7 @@ public class SliderAppMasterClient extends BaseHttpClient { quickLinks.put("JMX", entry.getValue().getAsString()); } else if ("org.apache.slider.monitor".equals(entry.getKey())) { quickLinks.put("UI", entry.getValue().getAsString()); - } else if ("org.apache.slider.metrics".equals(entry.getKey())) { + } else if ("app.metrics".equals(entry.getKey())) { quickLinks.put("Metrics", entry.getValue().getAsString()); } else { quickLinks.put(entry.getKey(), entry.getValue().getAsString()); @@ -148,17 +153,85 @@ public class SliderAppMasterClient extends BaseHttpClient { } } + public Map<String, Number[][]> getGangliaMetrics(String gangliaUrl, + Set<String> metricsRequested, + TemporalInfo temporalInfo, + ViewContext context, + SliderAppType appType, + MetricsHolder metricsHolder) { + Map<String, Number[][]> retVal = new HashMap<String, Number[][]>(); + + if (appType == null || metricsHolder == null || metricsHolder.getGangliaMetrics() == null) { + logger.info("AppType must be provided and it must contain ganglia_metrics.json to extract jmx properties"); + return retVal; + } + + Map<String, GangliaMetric> receivedMetrics = null; + List<String> components = new ArrayList<String>(); + for (SliderAppTypeComponent appTypeComponent : appType.getTypeComponents()) { + components.add(appTypeComponent.getName()); + } + + Map<String, Map<String, Map<String, Metric>>> metrics = metricsHolder.getGangliaMetrics(); + Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components); + Set<String> metricsToRead = new HashSet<String>(); + Map<String, String> reverseNameLookup = new HashMap<String, String>(); + for (String key : relevantMetrics.keySet()) { + if (metricsRequested.contains(key)) { + String metricName = relevantMetrics.get(key).getMetric(); + metricsToRead.add(metricName); + reverseNameLookup.put(metricName, key); + } + } + + if (metricsToRead.size() != 0) { + try { + String specWithParams = SliderAppGangliaHelper.getSpec(gangliaUrl, metricsToRead, temporalInfo); + logger.info("Using spec: " + specWithParams); + if (specWithParams != null) { + + String spec = null; + String params = null; + String[] tokens = specWithParams.split("\\?", 2); + + try { + spec = tokens[0]; + params = tokens[1]; + } catch (ArrayIndexOutOfBoundsException e) { + logger.info(e.toString()); + } + + receivedMetrics = SliderAppGangliaHelper.getGangliaMetrics(context, spec, params); + } + } catch (Exception e) { + logger.warn("Unable to retrieve ganglia metrics. " + e.getMessage()); + } + } + + if (receivedMetrics != null) { + for (GangliaMetric metric : receivedMetrics.values()) { + if (reverseNameLookup.containsKey(metric.getMetric_name())) { + retVal.put(reverseNameLookup.get(metric.getMetric_name()), metric.getDatapoints()); + } + } + } + + return retVal; + } + /** * Provides only the interesting JMX metric names and values. - * + * * @param jmxUrl - * + * * @return */ - public Map<String, String> getJmx(String jmxUrl, ViewContext context, - SliderAppType appType) { + public Map<String, String> getJmx(String jmxUrl, + ViewContext context, + SliderAppType appType, + MetricsHolder metricsHolder) { Map<String, String> jmxProperties = new HashMap<String, String>(); - if (appType == null || appType.getJmxMetrics() == null) { + if (appType == null || metricsHolder == null || metricsHolder.getJmxMetrics() == null) { logger .info("AppType must be provided and it must contain jmx_metrics.json to extract jmx properties"); return jmxProperties; @@ -169,12 +242,14 @@ public class SliderAppMasterClient extends BaseHttpClient { components.add(appTypeComponent.getName()); } - Map<String, Map<String, Map<String, Metric>>> metrics = appType - .getJmxMetrics(); - Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, - components); - SliderAppJmxHelper.JMXTypes jmxType = SliderAppJmxHelper - .jmxTypeExpected(relevantMetrics); + Map<String, Map<String, Map<String, Metric>>> metrics = metricsHolder.getJmxMetrics(); + Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components); + if (relevantMetrics.size() == 0) { + logger.info("No metrics found for components defined in the app."); + return jmxProperties; + } + + SliderAppJmxHelper.JMXTypes jmxType = SliderAppJmxHelper.jmxTypeExpected(relevantMetrics); if (jmxType == null) { logger .info("jmx_metrics.json is malformed. It may have mixed metric key types of unsupported metric key types."); @@ -195,20 +270,20 @@ public class SliderAppMasterClient extends BaseHttpClient { if (jmxStream != null) { switch (jmxType) { - case JMX_BEAN: - SliderAppJmxHelper.extractMetricsFromJmxBean(jmxStream, jmxUrl, - jmxProperties, relevantMetrics); - break; - case JSON: - SliderAppJmxHelper.extractMetricsFromJmxJson(jmxStream, jmxUrl, - jmxProperties, relevantMetrics); - break; - case XML: - SliderAppJmxHelper.extractMetricsFromJmxXML(jmxStream, jmxUrl, - jmxProperties, relevantMetrics); - break; - default: - logger.info("Unsupported jmx type."); + case JMX_BEAN: + SliderAppJmxHelper.extractMetricsFromJmxBean(jmxStream, jmxUrl, + jmxProperties, relevantMetrics); + break; + case JSON: + SliderAppJmxHelper.extractMetricsFromJmxJson(jmxStream, jmxUrl, + jmxProperties, relevantMetrics); + break; + case XML: + SliderAppJmxHelper.extractMetricsFromJmxXML(jmxStream, jmxUrl, + jmxProperties, relevantMetrics); + break; + default: + logger.info("Unsupported jmx type."); } } } catch (Exception e) {
