Repository: ambari Updated Branches: refs/heads/branch-1.6.0.slider 79f3a1a3d -> a1a9c4598
AMBARI-5924. Add basic ganglia metrics support for Slider view. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a1a9c459 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a1a9c459 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a1a9c459 Branch: refs/heads/branch-1.6.0.slider Commit: a1a9c4598e927a4110f72d40da8d106fa4ec38f1 Parents: 79f3a1a Author: Sumit Mohanty <[email protected]> Authored: Thu May 29 00:51:15 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Thu May 29 10:55:06 2014 -0700 ---------------------------------------------------------------------- .../apache/ambari/view/slider/SliderApp.java | 6 +- .../slider/SliderAppsViewControllerImpl.java | 64 +++--- .../rest/client/SliderAppGangliaHelper.java | 196 +++++++++++++++++++ .../rest/client/SliderAppMasterClient.java | 132 +++++++++---- 4 files changed, 337 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a1a9c459/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java index 9d748fa..db42484 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java @@ -38,7 +38,7 @@ public class SliderApp { private Map<String, String> urls; private Map<String, Map<String, String>> configs; private Map<String, SliderAppComponent> components; - private Map<String, Object> metrics; + private Map<String, Number[][]> metrics; public String getName() { return name; @@ -128,11 +128,11 @@ public class SliderApp { this.components = components; } - public Map<String, Object> getMetrics() { + public Map<String, Number[][]> getMetrics() { return metrics; } - public void setMetrics(Map<String, Object> metrics) { + public void setMetrics(Map<String, Number[][]> metrics) { this.metrics = metrics; } http://git-wip-us.apache.org/repos/asf/ambari/blob/a1a9c459/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java index 7ff7b82..e4cd15e 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java @@ -88,6 +88,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { private static final Logger logger = Logger .getLogger(SliderAppsViewControllerImpl.class); + private static String METRICS_PREFIX = "metrics/"; @Inject private ViewContext viewContext; @Inject @@ -307,10 +308,24 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } } if (properties != null && !properties.isEmpty()) { + SliderAppType matchedAppType = null; + List<SliderAppType> matchingAppTypes = getSliderAppTypes(null); + if (matchingAppTypes != null && matchingAppTypes.size() > 0) { + for (SliderAppType appType : matchingAppTypes) { + if ((appType.getTypeName() != null && appType.getTypeName().equalsIgnoreCase(app.getType())) && + (appType.getTypeVersion() != null + && appType.getTypeVersion().equalsIgnoreCase(app.getAppVersion()))) { + matchedAppType = appType; + break; + } + } + } + SliderAppMasterClient sliderAppClient = yarnApp.getTrackingUrl() == null ? null : new SliderAppMasterClient(yarnApp.getTrackingUrl()); SliderAppMasterData appMasterData = null; Map<String, String> quickLinks = new HashMap<String, String>(); + Set<String> gangliaMetrics = new HashSet<String>(); for (String property : properties) { if ("RUNNING".equals(app.getState())) { if (sliderAppClient != null) { @@ -334,23 +349,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } if (quickLinks != null && quickLinks.containsKey("JMX")) { String jmxUrl = quickLinks.get("JMX"); - List<SliderAppType> appTypes = getSliderAppTypes(null); - if (appTypes != null && appTypes.size() > 0) { - for (SliderAppType appType : appTypes) { - logger.info("TYPE: " + appType.getTypeName() + " " - + app.getType()); - logger.info("VERSION: " + appType.getTypeVersion() + " " - + app.getAppVersion()); - if ((appType.getTypeName() != null && appType.getTypeName() - .equalsIgnoreCase(app.getType())) - && (appType.getTypeVersion() != null && appType - .getTypeVersion().equalsIgnoreCase( - app.getAppVersion()))) { - app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, - appType)); - break; - } - } + if (matchedAppType != null) { + app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, matchedAppType)); } } Map<String, Map<String, String>> configs = sliderAppClient @@ -426,10 +426,23 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { + yarnApp.getName(), e); throw new RuntimeException(e.getMessage(), e); } + } else if (property.startsWith(METRICS_PREFIX)) { + gangliaMetrics.add(property.substring(METRICS_PREFIX.length())); } } } } + if (gangliaMetrics.size() > 0) { + if (quickLinks.isEmpty()) { + quickLinks = sliderAppClient + .getQuickLinks(appMasterData.publisherUrl); + } + if (quickLinks != null && quickLinks.containsKey("Metrics")) { + String metricsUrl = quickLinks.get("Metrics"); + app.setMetrics( + sliderAppClient.getGangliaMetrics(metricsUrl, gangliaMetrics, null, viewContext, matchedAppType)); + } + } } return app; } @@ -552,7 +565,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (sliderAppObject != null) { if (sliderAppsMap.containsKey(sliderAppObject.getName())) { if (sliderAppsMap.get(sliderAppObject.getName()).getId() - .compareTo(sliderAppObject.getId()) < 0) { + .compareTo(sliderAppObject.getId()) < 0) { sliderAppsMap.put(sliderAppObject.getName(), sliderAppObject); } } else { @@ -560,8 +573,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } } } - if (sliderAppsMap.size() > 0) + if (sliderAppsMap.size() > 0) { sliderApps.addAll(sliderAppsMap.values()); + } } finally { Thread.currentThread().setContextClassLoader(currentClassLoader); } @@ -713,7 +727,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { Map<String, Map<String, Map<String, Metric>>> metrics = null; try { InputStream inputStream = zipFile.getInputStream(zipFile - .getEntry("jmx_metrics.json")); + .getEntry(fileName)); ObjectMapper mapper = new ObjectMapper(); metrics = mapper.readValue(inputStream, @@ -874,11 +888,12 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { properties.add("id"); properties.add("name"); final SliderApp sliderApp = getSliderApp(appId, properties); - if (sliderApp == null) + if (sliderApp == null) { throw new ApplicationNotFoundException(appId); + } ApplicationId applicationId = UserGroupInformation.getBestUGI(null, - "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { + "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { public ApplicationId run() throws IOException, YarnException { SliderClient sliderClient = getSliderClient(); ActionFreezeArgs freezeArgs = new ActionFreezeArgs(); @@ -903,10 +918,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { properties.add("id"); properties.add("name"); final SliderApp sliderApp = getSliderApp(appId, properties); - if (sliderApp == null) + if (sliderApp == null) { throw new ApplicationNotFoundException(appId); + } ApplicationId applicationId = UserGroupInformation.getBestUGI(null, - "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { + "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { public ApplicationId run() throws IOException, YarnException { SliderClient sliderClient = getSliderClient(); ActionThawArgs thawArgs = new ActionThawArgs(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a1a9c459/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java new file mode 100644 index 0000000..34c4c90 --- /dev/null +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppGangliaHelper.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.slider.rest.client; + +import org.apache.ambari.server.controller.ganglia.GangliaMetric; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.spi.TemporalInfo; +import org.apache.ambari.view.ViewContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class SliderAppGangliaHelper { + + private static final Logger logger = Logger + .getLogger(SliderAppGangliaHelper.class); + + private static String getSetString(Set<String> set, int limit) { + StringBuilder sb = new StringBuilder(); + + if (limit == -1 || set.size() <= limit) { + for (String cluster : set) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(cluster); + } + } + return sb.toString(); + } + + private static Number convertToNumber(String s) { + return s.contains(".") ? Double.parseDouble(s) : Long.parseLong(s); + } + + public static Map<String, GangliaMetric> getGangliaMetrics(ViewContext context, + String spec, + String params) throws IOException { + Map<String, GangliaMetric> receivedMetrics = new HashMap<String, GangliaMetric>(); + Map<String, String> headers = new HashMap<String, String>(); + BufferedReader reader = new BufferedReader(new InputStreamReader( + context.getURLStreamProvider().readFrom(spec, "POST", params, headers))); + + String feedStart = reader.readLine(); + if (feedStart == null || feedStart.isEmpty()) { + logger.info("Empty feed while getting ganglia metrics for spec => " + + spec); + return null; + } + int startTime = convertToNumber(feedStart).intValue(); + + String dsName = reader.readLine(); + if (dsName == null || dsName.isEmpty()) { + logger.info("Feed without body while reading ganglia metrics for spec " + + "=> " + spec); + return null; + } + + while (!"[~EOF]".equals(dsName)) { + GangliaMetric metric = new GangliaMetric(); + List<GangliaMetric.TemporalMetric> listTemporalMetrics = + new ArrayList<GangliaMetric.TemporalMetric>(); + + metric.setDs_name(dsName); + metric.setCluster_name(reader.readLine()); + metric.setHost_name(reader.readLine()); + metric.setMetric_name(reader.readLine()); + + String timeStr = reader.readLine(); + String stepStr = reader.readLine(); + if (timeStr == null || timeStr.isEmpty() || stepStr == null + || stepStr.isEmpty()) { + logger.info("Unexpected end of stream reached while getting ganglia " + + "metrics for spec => " + spec); + return null; + } + int time = convertToNumber(timeStr).intValue(); + int step = convertToNumber(stepStr).intValue(); + + String val = reader.readLine(); + String lastVal = null; + + while (val != null && !"[~EOM]".equals(val)) { + if (val.startsWith("[~r]")) { + Integer repeat = Integer.valueOf(val.substring(4)) - 1; + for (int i = 0; i < repeat; ++i) { + if (!"[~n]".equals(lastVal)) { + GangliaMetric.TemporalMetric tm = new GangliaMetric.TemporalMetric(lastVal, time); + if (tm.isValid()) listTemporalMetrics.add(tm); + } + time += step; + } + } else { + if (!"[~n]".equals(val)) { + GangliaMetric.TemporalMetric tm = new GangliaMetric.TemporalMetric(val, time); + if (tm.isValid()) listTemporalMetrics.add(tm); + } + time += step; + } + lastVal = val; + val = reader.readLine(); + } + + metric.setDatapointsFromList(listTemporalMetrics); + receivedMetrics.put(metric.getMetric_name(), metric); + + dsName = reader.readLine(); + if (dsName == null || dsName.isEmpty()) { + logger.info("Unexpected end of stream reached while getting ganglia " + + "metrics for spec => " + spec); + return null; + } + } + String feedEnd = reader.readLine(); + if (feedEnd == null || feedEnd.isEmpty()) { + logger.info("Error reading end of feed while getting ganglia metrics " + + "for spec => " + spec); + } else { + + int endTime = convertToNumber(feedEnd).intValue(); + int totalTime = endTime - startTime; + if (logger.isInfoEnabled() && totalTime > 3) { + logger.info("Ganglia resource population time: " + totalTime); + } + } + return receivedMetrics; + } + + public static String getSpec(String gangliaUrl, + Set<String> metricSet, + TemporalInfo temporalInfo) throws SystemException, URISyntaxException { + + String metrics = getSetString(metricSet, -1); + + URIBuilder uriBuilder = new URIBuilder(gangliaUrl); + + uriBuilder.setParameter("h", "__SummaryInfo__"); + + if (metrics.length() > 0) { + uriBuilder.setParameter("m", metrics); + } else { + // get all metrics + uriBuilder.setParameter("m", ".*"); + } + + if (temporalInfo != null) { + long startTime = temporalInfo.getStartTime(); + if (startTime != -1) { + uriBuilder.setParameter("s", String.valueOf(startTime)); + } + + long endTime = temporalInfo.getEndTime(); + if (endTime != -1) { + uriBuilder.setParameter("e", String.valueOf(endTime)); + } + + long step = temporalInfo.getStep(); + if (step != -1) { + uriBuilder.setParameter("r", String.valueOf(step)); + } + } else { + long endTime = System.currentTimeMillis()/1000; + long startTime = System.currentTimeMillis()/1000 - 60*60; + uriBuilder.setParameter("e", String.valueOf(endTime)); + uriBuilder.setParameter("s", String.valueOf(startTime)); + uriBuilder.setParameter("r", "15"); + } + + return uriBuilder.toString(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a1a9c459/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java index d77c1c7..7cfc4b1 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java @@ -18,14 +18,10 @@ package org.apache.ambari.view.slider.rest.client; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.apache.ambari.server.controller.ganglia.GangliaMetric; +import org.apache.ambari.server.controller.spi.TemporalInfo; import org.apache.ambari.view.URLStreamProvider; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.slider.SliderAppType; @@ -33,8 +29,15 @@ import org.apache.ambari.view.slider.SliderAppTypeComponent; import org.apache.commons.httpclient.HttpException; import org.apache.log4j.Logger; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; public class SliderAppMasterClient extends BaseHttpClient { @@ -122,7 +125,7 @@ public class SliderAppMasterClient extends BaseHttpClient { continue; } JsonElement entryJson = super.doGetJson(providerUrl, "/slider/" - + entry.getKey()); + + entry.getKey()); if (entryJson != null) { JsonObject configsObj = entryJson.getAsJsonObject().get("entries") .getAsJsonObject(); @@ -146,15 +149,74 @@ public class SliderAppMasterClient extends BaseHttpClient { } } + public Map<String, Number[][]> getGangliaMetrics(String gangliaUrl, + Set<String> metricsRequested, + TemporalInfo temporalInfo, + ViewContext context, + SliderAppType appType) { + Map<String, Number[][]> retVal = new HashMap<String, Number[][]>(); + Map<String, GangliaMetric> receivedMetrics = null; + List<String> components = new ArrayList<String>(); + for (SliderAppTypeComponent appTypeComponent : appType.getTypeComponents()) { + components.add(appTypeComponent.getName()); + } + + Map<String, Map<String, Map<String, Metric>>> metrics = appType.getGangliaMetrics(); + Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components); + Set<String> metricsToRead = new HashSet<String>(); + Map<String, String> reverseNameLookup = new HashMap<String, String>(); + for (String key : relevantMetrics.keySet()) { + if (metricsRequested.contains(key)) { + String metricName = relevantMetrics.get(key).getMetric(); + metricsToRead.add(metricName); + reverseNameLookup.put(metricName, key); + } + } + + if (metricsToRead.size() != 0) { + try { + String specWithParams = SliderAppGangliaHelper.getSpec(gangliaUrl, metricsToRead, temporalInfo); + logger.info("Using spec: " + specWithParams); + if (specWithParams != null) { + + String spec = null; + String params = null; + String[] tokens = specWithParams.split("\\?", 2); + + try { + spec = tokens[0]; + params = tokens[1]; + } catch (ArrayIndexOutOfBoundsException e) { + logger.info(e.toString()); + } + + receivedMetrics = SliderAppGangliaHelper.getGangliaMetrics(context, spec, params); + } + } catch (Exception e) { + logger.warn("Unable to retrieve ganglia metrics. " + e.getMessage()); + } + } + + if (receivedMetrics != null) { + for (GangliaMetric metric : receivedMetrics.values()) { + if (reverseNameLookup.containsKey(metric.getMetric_name())) { + retVal.put(reverseNameLookup.get(metric.getMetric_name()), metric.getDatapoints()); + } + } + } + + return retVal; + } + /** * Provides only the interesting JMX metric names and values. - * + * * @param jmxUrl - * + * * @return */ public Map<String, String> getJmx(String jmxUrl, ViewContext context, - SliderAppType appType) { + SliderAppType appType) { Map<String, String> jmxProperties = new HashMap<String, String>(); if (appType == null || appType.getJmxMetrics() == null) { logger @@ -167,12 +229,14 @@ public class SliderAppMasterClient extends BaseHttpClient { components.add(appTypeComponent.getName()); } - Map<String, Map<String, Map<String, Metric>>> metrics = appType - .getJmxMetrics(); - Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, - components); - SliderAppJmxHelper.JMXTypes jmxType = SliderAppJmxHelper - .jmxTypeExpected(relevantMetrics); + Map<String, Map<String, Map<String, Metric>>> metrics = appType.getJmxMetrics(); + Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components); + if (relevantMetrics.size() == 0) { + logger.info("No metrics found for components defined in the app."); + return jmxProperties; + } + + SliderAppJmxHelper.JMXTypes jmxType = SliderAppJmxHelper.jmxTypeExpected(relevantMetrics); if (jmxType == null) { logger .info("jmx_metrics.json is malformed. It may have mixed metric key types of unsupported metric key types."); @@ -193,20 +257,20 @@ public class SliderAppMasterClient extends BaseHttpClient { if (jmxStream != null) { switch (jmxType) { - case JMX_BEAN: - SliderAppJmxHelper.extractMetricsFromJmxBean(jmxStream, jmxUrl, - jmxProperties, relevantMetrics); - break; - case JSON: - SliderAppJmxHelper.extractMetricsFromJmxJson(jmxStream, jmxUrl, - jmxProperties, relevantMetrics); - break; - case XML: - SliderAppJmxHelper.extractMetricsFromJmxXML(jmxStream, jmxUrl, - jmxProperties, relevantMetrics); - break; - default: - logger.info("Unsupported jmx type."); + case JMX_BEAN: + SliderAppJmxHelper.extractMetricsFromJmxBean(jmxStream, jmxUrl, + jmxProperties, relevantMetrics); + break; + case JSON: + SliderAppJmxHelper.extractMetricsFromJmxJson(jmxStream, jmxUrl, + jmxProperties, relevantMetrics); + break; + case XML: + SliderAppJmxHelper.extractMetricsFromJmxXML(jmxStream, jmxUrl, + jmxProperties, relevantMetrics); + break; + default: + logger.info("Unsupported jmx type."); } } } catch (Exception e) {
