Repository: ambari Updated Branches: refs/heads/trunk 53e3d8dd0 -> d1c4a93fe
AMBARI-5920. Add support to parse jmx metrics in the Slider app view Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d1c4a93f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d1c4a93f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d1c4a93f Branch: refs/heads/trunk Commit: d1c4a93fe651132b29b26c230b960ddd7b9b3362 Parents: 53e3d8d Author: Sumit Mohanty <[email protected]> Authored: Wed May 28 16:51:48 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed May 28 16:52:09 2014 -0700 ---------------------------------------------------------------------- .../apache/ambari/view/slider/SliderApp.java | 9 + .../slider/SliderAppsViewControllerImpl.java | 183 +++++++++-------- .../ambari/view/slider/rest/client/Metric.java | 29 ++- .../slider/rest/client/SliderAppJmxHelper.java | 205 +++++++++++++++++++ .../rest/client/SliderAppMasterClient.java | 109 +++------- .../rest/client/SliderAppMasterClientTest.java | 111 ++++++++-- 6 files changed, 463 insertions(+), 183 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d1c4a93f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java index 7d2c4be..9d748fa 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderApp.java @@ -38,6 +38,7 @@ public class SliderApp { private Map<String, String> urls; private Map<String, Map<String, String>> configs; private Map<String, SliderAppComponent> components; + private Map<String, Object> metrics; public String getName() { return name; @@ -127,6 +128,14 @@ public class SliderApp { this.components = components; } + public Map<String, Object> getMetrics() { + return metrics; + } + + public void setMetrics(Map<String, Object> metrics) { + this.metrics = metrics; + } + public long getStartTime() { return startTime; } http://git-wip-us.apache.org/repos/asf/ambari/blob/d1c4a93f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java index 4ccc51b..c6a26ca 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java @@ -18,22 +18,13 @@ package org.apache.ambari.view.slider; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Field; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.zip.ZipException; - +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.inject.Inject; +import com.google.inject.Singleton; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.slider.clients.AmbariClient; import org.apache.ambari.view.slider.clients.AmbariCluster; @@ -75,13 +66,21 @@ import org.apache.tools.zip.ZipFile; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.inject.Inject; -import com.google.inject.Singleton; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.zip.ZipException; @Singleton public class SliderAppsViewControllerImpl implements SliderAppsViewController { @@ -93,13 +92,12 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { @Inject private AmbariClient ambariClient; private List<SliderAppType> appTypes; - private Integer createAppCounter = -1; private String getAppsFolderPath() { return viewContext - .getAmbariProperty(org.apache.ambari.server.configuration.Configuration.RESOURCES_DIR_KEY) - + "/apps"; + .getAmbariProperty(org.apache.ambari.server.configuration.Configuration.RESOURCES_DIR_KEY) + + "/apps"; } private String getAppsCreateFolderPath() { @@ -193,7 +191,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { private String getApplicationIdString(ApplicationId appId) { return Long.toString(appId.getClusterTimestamp()) + "_" - + Integer.toString(appId.getId()); + + Integer.toString(appId.getId()); } private ApplicationId getApplicationId(String appIdString) { @@ -229,7 +227,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } private SliderApp createSliderAppObject(ApplicationReport yarnApp, - Set<String> properties, SliderClient sliderClient) { + Set<String> properties, SliderClient sliderClient) { if (yarnApp == null) { return null; } @@ -271,12 +269,13 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (index > 0 && index < tag.length() - 1) { String key = tag.substring(0, index).trim(); String value = tag.substring(index + 1).trim(); - if ("name".equals(key)) + if ("name".equals(key)) { app.setType(value); - else if ("version".equals(key)) + } else if ("version".equals(key)) { app.setAppVersion(value); - else if ("description".equals(key)) + } else if ("description".equals(key)) { app.setDescription(value); + } } } } @@ -310,10 +309,17 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { String jmxUrl = quickLinks.get("JMX"); List<SliderAppType> appTypes = getSliderAppTypes(null); if (appTypes != null && appTypes.size() > 0) { - // TODO: Get the correct app type based on name and version - SliderAppType appType = appTypes.get(0); - app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, - appType)); + for (SliderAppType appType : appTypes) { + logger.info("TYPE: " + appType.getTypeName() + " " + app.getType()); + logger.info("VERSION: " + appType.getTypeVersion() + " " + app.getAppVersion()); + if ((appType.getTypeName() != null && appType.getTypeName().equalsIgnoreCase(app.getType())) && + (appType.getTypeVersion() != null + && appType.getTypeVersion().equalsIgnoreCase(app.getAppVersion()))) { + app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, + appType)); + break; + } + } } } Map<String, Map<String, String>> configs = sliderAppClient @@ -329,8 +335,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { Map<String, SliderAppComponent> componentTypeMap = new HashMap<String, SliderAppComponent>(); for (Entry<String, Object> e : description.status.entrySet()) { @SuppressWarnings("unchecked") - Map<String, Map<String, Map<String, Object>>> componentsObj = (Map<String, Map<String, Map<String, Object>>>) e - .getValue(); + Map<String, Map<String, Map<String, Object>>> + componentsObj = + (Map<String, Map<String, Map<String, Object>>>) e + .getValue(); boolean isLive = "live".equals(e.getKey()); for (Entry<String, Map<String, Map<String, Object>>> componentEntry : componentsObj .entrySet()) { @@ -344,7 +352,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { appComponent .setCompletedContainers(new HashMap<String, Map<String, String>>()); componentTypeMap.put(componentEntry.getKey(), - appComponent); + appComponent); } for (Entry<String, Map<String, Object>> containerEntry : componentEntry .getValue().entrySet()) { @@ -357,19 +365,19 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { Object containerPropertyValue = containerValues .get(containerProperty); containerDataMap.put(containerProperty, - containerPropertyValue.toString()); + containerPropertyValue.toString()); } if (isLive) { appComponent.getActiveContainers().put(containerId, - containerDataMap); + containerDataMap); } else { appComponent.getCompletedContainers().put( containerId, containerDataMap); } } appComponent.setInstanceCount(appComponent - .getActiveContainers().size() - + appComponent.getCompletedContainers().size()); + .getActiveContainers().size() + + appComponent.getCompletedContainers().size()); } } app.setComponents(componentTypeMap); @@ -377,16 +385,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } catch (UnknownApplicationInstanceException e) { logger.warn( "Unable to determine app components for " - + yarnApp.getName(), e); + + yarnApp.getName(), e); } catch (YarnException e) { logger.warn( "Unable to determine app components for " - + yarnApp.getName(), e); + + yarnApp.getName(), e); throw new RuntimeException(e.getMessage(), e); } catch (IOException e) { logger.warn( "Unable to determine app components for " - + yarnApp.getName(), e); + + yarnApp.getName(), e); throw new RuntimeException(e.getMessage(), e); } } @@ -398,10 +406,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } /** - * Creates a new {@link SliderClient} initialized with appropriate - * configuration. If configuration was not determined, <code>null</code> is - * returned. - * + * Creates a new {@link SliderClient} initialized with appropriate configuration. If configuration was not determined, + * <code>null</code> is returned. + * * @return */ protected SliderClient getSliderClient() { @@ -418,7 +425,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { super.serviceInit(conf); // Override the default FS client to set the super user. FileSystem fs = FileSystem.get(FileSystem.getDefaultUri(getConfig()), - getConfig(), "yarn"); + getConfig(), "yarn"); SliderFileSystem fileSystem = new SliderFileSystem(fs, getConfig()); Field fsField = SliderClient.class .getDeclaredField("sliderFileSystem"); @@ -428,7 +435,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { }; try { sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, - new String[] { "usage" }); + new String[]{"usage"}); } catch (Exception e) { logger.warn("Unable to set SliderClient configs", e); throw new RuntimeException(e.getMessage(), e); @@ -441,16 +448,15 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } /** - * Dynamically determines Slider client configuration. If unable to determine, - * <code>null</code> is returned. - * + * Dynamically determines Slider client configuration. If unable to determine, <code>null</code> is returned. + * * @return */ private Configuration getSliderClientConfiguration() { AmbariCluster ambariCluster = getAmbariCluster(); if (ambariCluster != null) { AmbariService zkService = ambariClient.getService(ambariCluster, - "ZOOKEEPER"); + "ZOOKEEPER"); if (zkService != null && ambariCluster.getDesiredConfigs() != null && ambariCluster.getDesiredConfigs().containsKey("global") && ambariCluster.getDesiredConfigs().containsKey("yarn-site") @@ -485,7 +491,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { yarnConfig.set("yarn.log-aggregation-enable", "true"); yarnConfig.set("yarn.resourcemanager.address", rmAddress); yarnConfig.set("yarn.resourcemanager.scheduler.address", - rmSchedulerAddress); + rmSchedulerAddress); yarnConfig.set("fs.defaultFS", hdfsPath); yarnConfig.set("slider.zookeeper.quorum", zkQuorum.toString()); yarnConfig @@ -510,7 +516,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { List<ApplicationReport> yarnApps = sliderClient.listSliderInstances(null); for (ApplicationReport yarnApp : yarnApps) { SliderApp sliderAppObject = createSliderAppObject(yarnApp, properties, - sliderClient); + sliderClient); if (sliderAppObject != null) { sliderApps.add(sliderAppObject); } @@ -577,7 +583,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { try { ZipFile zipFile = new ZipFile(appZip); Metainfo metainfo = new MetainfoParser().parse(zipFile - .getInputStream(zipFile.getEntry("metainfo.xml"))); + .getInputStream(zipFile.getEntry("metainfo.xml"))); // Create app type object if (metainfo.getServices() != null && metainfo.getServices().size() > 0) { @@ -609,8 +615,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { // Components ArrayList<SliderAppTypeComponent> appTypeComponentList = new ArrayList<SliderAppTypeComponent>(); for (Component component : service.getComponents()) { - if ("CLIENT".equals(component.getCategory())) + if ("CLIENT".equals(component.getCategory())) { continue; + } SliderAppTypeComponent appTypeComponent = new SliderAppTypeComponent(); appTypeComponent.setDisplayName(component.getName()); appTypeComponent.setId(component.getName()); @@ -620,11 +627,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { // appTypeComponent.setPriority(component.); if (component.getMinInstanceCount() != null) { appTypeComponent.setInstanceCount(Integer.parseInt(component - .getMinInstanceCount())); + .getMinInstanceCount())); } if (component.getMaxInstanceCount() != null) { appTypeComponent.setMaxInstanceCount(Integer - .parseInt(component.getMaxInstanceCount())); + .parseInt(component.getMaxInstanceCount())); } if (resourcesJson != null) { JsonElement componentJson = resourcesJson.getAsJsonObject() @@ -632,10 +639,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { .get(component.getName()); if (componentJson != null && componentJson.getAsJsonObject().has( - "yarn.role.priority")) { + "yarn.role.priority")) { appTypeComponent.setPriority(Integer.parseInt(componentJson - .getAsJsonObject().get("yarn.role.priority") - .getAsString())); + .getAsJsonObject().get("yarn.role.priority") + .getAsString())); } } appTypeComponent.setCategory(component.getCategory()); @@ -644,7 +651,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { appType.setJmxMetrics(readMetrics(zipFile, "jmx_metrics.json")); appType.setGangliaMetrics(readMetrics(zipFile, - "ganglia_metrics.json")); + "ganglia_metrics.json")); appType.setTypeComponents(appTypeComponentList); appTypes.add(appType); @@ -661,16 +668,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } Map<String, Map<String, Map<String, Metric>>> readMetrics(ZipFile zipFile, - String fileName) { + String fileName) { Map<String, Map<String, Map<String, Metric>>> metrics = null; try { InputStream inputStream = zipFile.getInputStream(zipFile - .getEntry("jmx_metrics.json")); + .getEntry("jmx_metrics.json")); ObjectMapper mapper = new ObjectMapper(); metrics = mapper.readValue(inputStream, - new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() { - }); + new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() { + }); } catch (IOException e) { logger.info("Error reading metrics. " + e.getMessage()); } @@ -688,8 +695,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { JsonArray componentsArray = json.get("typeComponents").getAsJsonArray(); String appsCreateFolderPath = getAppsCreateFolderPath(); File appsCreateFolder = new File(appsCreateFolderPath); - if (!appsCreateFolder.exists()) + if (!appsCreateFolder.exists()) { appsCreateFolder.mkdirs(); + } int appCount; synchronized (createAppCounter) { if (createAppCounter < 0) { @@ -699,8 +707,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { for (String app : apps) { try { int count = Integer.parseInt(app); - if (count > createAppCounter) + if (count > createAppCounter) { createAppCounter = count; + } } catch (NumberFormatException e) { } } @@ -708,7 +717,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { appCount = ++createAppCounter; } File appCreateFolder = new File(appsCreateFolder, - Integer.toString(appCount)); + Integer.toString(appCount)); appCreateFolder.mkdirs(); File appConfigJsonFile = new File(appCreateFolder, "appConfig.json"); File resourcesJsonFile = new File(appCreateFolder, "resources.json"); @@ -719,28 +728,29 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { AmbariCluster cluster = ambariClient.getCluster(clusterInfo); Map<String, String> coreSiteConfigs = ambariClient.getConfiguration( clusterInfo, "core-site", cluster.getDesiredConfigs() - .get("core-site")); + .get("core-site")); String hdfsLocation = coreSiteConfigs.get("fs.defaultFS"); final ActionCreateArgs createArgs = new ActionCreateArgs(); createArgs.template = appConfigJsonFile; createArgs.resources = resourcesJsonFile; createArgs.image = new Path(hdfsLocation - + "/slider/agent/slider-agent.tar.gz"); + + "/slider/agent/slider-agent.tar.gz"); ClassLoader currentClassLoader = Thread.currentThread() .getContextClassLoader(); Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); try { ApplicationId applicationId = UserGroupInformation.getBestUGI(null, - "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { + "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { public ApplicationId run() throws IOException, YarnException { SliderClient sliderClient = getSliderClient(); sliderClient.actionCreate(appName, createArgs); return sliderClient.applicationId; } }); - if (applicationId != null) + if (applicationId != null) { return getApplicationIdString(applicationId); + } } finally { Thread.currentThread().setContextClassLoader(currentClassLoader); } @@ -749,10 +759,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } private void saveAppResources(JsonArray componentsArray, - File resourcesJsonFile) throws IOException { + File resourcesJsonFile) throws IOException { JsonObject resourcesObj = new JsonObject(); resourcesObj.addProperty("schema", - "http://example.org/specification/v2.0.0";); + "http://example.org/specification/v2.0.0";); resourcesObj.add("metadata", new JsonObject()); resourcesObj.add("global", new JsonObject()); JsonObject componentsObj = new JsonObject(); @@ -762,11 +772,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (inputComponent.has("id")) { JsonObject componentValue = new JsonObject(); componentValue.addProperty("yarn.role.priority", - inputComponent.get("priority").getAsString()); + inputComponent.get("priority").getAsString()); componentValue.addProperty("yarn.component.instances", inputComponent .get("instanceCount").getAsString()); componentsObj.add(inputComponent.get("id").getAsString(), - componentValue); + componentValue); } } } @@ -777,13 +787,14 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { fos = new FileOutputStream(resourcesJsonFile); IOUtils.write(jsonString, fos); } finally { - if (fos != null) + if (fos != null) { fos.close(); + } } } private void saveAppConfigs(JsonObject configs, JsonArray componentsArray, - File appConfigJsonFile) throws IOException { + File appConfigJsonFile) throws IOException { JsonObject appConfigs = new JsonObject(); appConfigs.addProperty("schema", "http://example.org/specification/v2.0.0";); appConfigs.add("metadata", new JsonObject()); @@ -792,9 +803,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (componentsArray != null) { for (int i = 0; i < componentsArray.size(); i++) { JsonObject inputComponent = componentsArray.get(i).getAsJsonObject(); - if (inputComponent.has("id")) + if (inputComponent.has("id")) { componentsObj.add(inputComponent.get("id").getAsString(), - new JsonObject()); + new JsonObject()); + } } } appConfigs.add("components", componentsObj); @@ -804,8 +816,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { fos = new FileOutputStream(appConfigJsonFile); IOUtils.write(jsonString, fos); } finally { - if (fos != null) + if (fos != null) { fos.close(); + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/d1c4a93f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/Metric.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/Metric.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/Metric.java index a9692c0..e92d174 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/Metric.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/Metric.java @@ -18,10 +18,18 @@ package org.apache.ambari.view.slider.rest.client; +import org.apache.log4j.Logger; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; import java.util.ArrayList; import java.util.List; public class Metric { + private static final Logger logger = Logger + .getLogger(Metric.class); private static String SEPARATOR = "."; private static char SEPARATOR_REPLACED = '#'; private String metric; @@ -29,6 +37,8 @@ public class Metric { private boolean temporal; private String keyName = null; private List<List<String>> matchers = null; + private XPathExpression xPathExpression = null; + private boolean xPathExpressionComputed = false; private Metric() { } @@ -63,7 +73,24 @@ public class Metric { this.temporal = temporal; } - public String getKeyName() { + public XPathExpression getxPathExpression() { + if (!xPathExpressionComputed) { + XPathFactory xPathfactory = XPathFactory.newInstance(); + XPath xpath = xPathfactory.newXPath(); + XPathExpression schemaPath = null; + try { + schemaPath = xpath.compile(metric); + } catch (XPathExpressionException e) { + logger.info(String.format("Unable to compile %s into xpath expression", metric)); + } + xPathExpression = schemaPath; + xPathExpressionComputed = true; + } + + return xPathExpression; + } + + public String getJmxBeanKeyName() { if (keyName == null) { int firstIndex = metric.indexOf(SEPARATOR); if (firstIndex > 0) { http://git-wip-us.apache.org/repos/asf/ambari/blob/d1c4a93f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppJmxHelper.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppJmxHelper.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppJmxHelper.java new file mode 100644 index 0000000..e0a8072 --- /dev/null +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppJmxHelper.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.slider.rest.client; + +import org.apache.ambari.server.controller.jmx.JMXMetricHolder; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Logger; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectReader; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SliderAppJmxHelper { + + private static final Logger logger = Logger + .getLogger(SliderAppJmxHelper.class); + private static final String NAME_KEY = "name"; + private static final String PORT_KEY = "tag.port"; + private static final String FORPORT_KEY = "ForPort"; + private static final String JSON_METRIC_START = "$['"; + private static final String JSON_METRIC_END = "']"; + + public static JMXTypes jmxTypeExpected(Map<String, Metric> metrics) { + JMXTypes retVal = null; + for (Metric metric : metrics.values()) { + if (retVal == null) { + retVal = getMetricType(metric.getMetric()); + continue; + } else { + if (retVal != getMetricType(metric.getMetric())) { + retVal = null; + break; + } + } + } + return retVal; + } + + public static void extractMetricsFromJmxBean(InputStream jmxStream, String jmxUrl, + Map<String, String> jmxProperties, + Map<String, Metric> metrics) { + ObjectMapper jmxObjectMapper = new ObjectMapper(); + jmxObjectMapper.configure(DeserializationConfig.Feature.USE_ANNOTATIONS, false); + ObjectReader jmxObjectReader = jmxObjectMapper.reader(JMXMetricHolder.class); + JMXMetricHolder metricHolder = null; + try { + metricHolder = jmxObjectReader.readValue(jmxStream); + } catch (IOException e) { + logger.error(String.format("Malformed jmx data from %s. Error %s", jmxUrl, e.getMessage())); + } + + Map<String, Map<String, Object>> categories = new HashMap<String, Map<String, Object>>(); + + for (Map<String, Object> bean : metricHolder.getBeans()) { + String category = getCategory(bean); + if (category != null) { + categories.put(category, bean); + } + } + + addJmxPropertiesFromBeans(jmxProperties, categories, metrics); + } + + public static void extractMetricsFromJmxJson(InputStream jmxStream, String jmxUrl, + Map<String, String> jmxProperties, + Map<String, Metric> metrics) + throws IOException, ParseException { + JSONParser parser = new JSONParser(); + Object obj = parser.parse(IOUtils.toString(jmxStream)); + JSONObject jsonObject = (JSONObject) obj; + for (String key : metrics.keySet()) { + Metric metric = metrics.get(key); + String jsonKey = extractJsonKeySingleLevel(metric.getMetric()); + Object value = jsonObject.get(jsonKey); + if (value != null) { + jmxProperties.put(key, value.toString()); + } + } + } + + private static String extractJsonKeySingleLevel(String metricKey) { + String jsonKey = metricKey; + if (metricKey != null) { + if (metricKey.startsWith(JSON_METRIC_START) && metricKey.endsWith(JSON_METRIC_END)) { + jsonKey = metricKey.substring(JSON_METRIC_START.length(), metricKey.length() - JSON_METRIC_END.length()); + } + } + + return jsonKey; + } + + public static void extractMetricsFromJmxXML(InputStream jmxStream, String jmxUrl, + Map<String, String> jmxProperties, + Map<String, Metric> metrics) + throws ParserConfigurationException, IOException, SAXException, XPathExpressionException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(jmxStream); + for (String key : metrics.keySet()) { + Metric metric = metrics.get(key); + XPathExpression xPathExpression = metric.getxPathExpression(); + if (xPathExpression != null) { + String value = xPathExpression.evaluate(doc); + if (value != null) { + jmxProperties.put(key, value.toString().trim()); + } + } + } + } + + private static String getCategory(Map<String, Object> bean) { + if (bean.containsKey(NAME_KEY)) { + String name = (String) bean.get(NAME_KEY); + + if (bean.containsKey(PORT_KEY)) { + String port = (String) bean.get(PORT_KEY); + name = name.replace(FORPORT_KEY + port, ""); + } + return name; + } + return null; + } + + protected static void addJmxPropertiesFromBeans(Map<String, String> jmxProperties, + Map<String, Map<String, Object>> categories, + Map<String, Metric> relevantMetrics) { + for (String metricName : relevantMetrics.keySet()) { + Metric metric = relevantMetrics.get(metricName); + String beanName = metric.getJmxBeanKeyName(); + Object value = categories.get(beanName); + if (value instanceof Map) { + Map<?, ?> map = (Map<?, ?>) value; + for (List<String> matcher : metric.getMatchers()) { + boolean foundMetrics = false; + for (int matchIndex = 0; matchIndex < matcher.size(); matchIndex++) { + String matchKey = matcher.get(matchIndex); + value = map.get(matchKey); + if (value instanceof Map) { + map = (Map<?, ?>) value; + continue; + } else { + if (value != null && matchIndex == matcher.size() - 1) { + jmxProperties.put(metricName, value.toString()); + foundMetrics = true; + } else { + break; + } + } + } + if (foundMetrics) { + break; + } + } + } + } + } + + private static JMXTypes getMetricType(String metricKey) { + assert metricKey != null; + if (metricKey.startsWith("/")) { + return JMXTypes.XML; + } else if (metricKey.startsWith("$")) { + return JMXTypes.JSON; + } else { + return JMXTypes.JMX_BEAN; + } + } + + public enum JMXTypes { + JMX_BEAN, + JSON, + XML + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/d1c4a93f/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java index 99b22ff..9318128 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java @@ -20,16 +20,12 @@ package org.apache.ambari.view.slider.rest.client; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import org.apache.ambari.server.controller.jmx.JMXMetricHolder; import org.apache.ambari.view.URLStreamProvider; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.slider.SliderAppType; import org.apache.ambari.view.slider.SliderAppTypeComponent; import org.apache.commons.httpclient.HttpException; import org.apache.log4j.Logger; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.ObjectReader; import java.io.IOException; import java.io.InputStream; @@ -43,9 +39,6 @@ public class SliderAppMasterClient extends BaseHttpClient { private static final Logger logger = Logger .getLogger(SliderAppMasterClient.class); - private static final String NAME_KEY = "name"; - private static final String PORT_KEY = "tag.port"; - private static final String FORPORT_KEY = "ForPort"; public SliderAppMasterClient(String url) { super(url); @@ -158,8 +151,21 @@ public class SliderAppMasterClient extends BaseHttpClient { */ public Map<String, String> getJmx(String jmxUrl, ViewContext context, SliderAppType appType) { Map<String, String> jmxProperties = new HashMap<String, String>(); - if (appType == null) { - logger.info("AppType must be provided to extract jmx properties"); + if (appType == null || appType.getJmxMetrics() == null) { + logger.info("AppType must be provided and it must contain jmx_metrics.json to extract jmx properties"); + return jmxProperties; + } + + List<String> components = new ArrayList<String>(); + for (SliderAppTypeComponent appTypeComponent : appType.getTypeComponents()) { + components.add(appTypeComponent.getName()); + } + + Map<String, Map<String, Map<String, Metric>>> metrics = appType.getJmxMetrics(); + Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components); + SliderAppJmxHelper.JMXTypes jmxType = SliderAppJmxHelper.jmxTypeExpected(relevantMetrics); + if (jmxType == null) { + logger.info("jmx_metrics.json is malformed. It may have mixed metric key types of unsupported metric key types."); return jmxProperties; } @@ -174,33 +180,19 @@ public class SliderAppMasterClient extends BaseHttpClient { } if (jmxStream != null) { - ObjectMapper jmxObjectMapper = new ObjectMapper(); - jmxObjectMapper.configure(DeserializationConfig.Feature.USE_ANNOTATIONS, false); - ObjectReader jmxObjectReader = jmxObjectMapper.reader(JMXMetricHolder.class); - JMXMetricHolder metricHolder = null; - try { - metricHolder = jmxObjectReader.readValue(jmxStream); - } catch (IOException e) { - logger.error(String.format("Malformed jmx data from %s. Error %s", jmxUrl, e.getMessage())); - } - - Map<String, Map<String, Object>> categories = new HashMap<String, Map<String, Object>>(); - - for (Map<String, Object> bean : metricHolder.getBeans()) { - String category = getCategory(bean); - if (category != null) { - categories.put(category, bean); - } - } - - List<String> components = new ArrayList<String>(); - for (SliderAppTypeComponent appTypeComponent : appType.getTypeComponents()) { - components.add(appTypeComponent.getName()); + switch (jmxType) { + case JMX_BEAN: + SliderAppJmxHelper.extractMetricsFromJmxBean(jmxStream, jmxUrl, jmxProperties, relevantMetrics); + break; + case JSON: + SliderAppJmxHelper.extractMetricsFromJmxJson(jmxStream, jmxUrl, jmxProperties, relevantMetrics); + break; + case XML: + SliderAppJmxHelper.extractMetricsFromJmxXML(jmxStream, jmxUrl, jmxProperties, relevantMetrics); + break; + default: + logger.info("Unsupported jmx type."); } - - Map<String, Map<String, Map<String, Metric>>> metrics = appType.getJmxMetrics(); - Map<String, Metric> relevantMetrics = getRelevantMetrics(metrics, components); - addJmxProperties(jmxProperties, categories, relevantMetrics); } } catch (Exception e) { logger.info("Failed to extract jmx metrics. " + e.getMessage()); @@ -209,40 +201,6 @@ public class SliderAppMasterClient extends BaseHttpClient { return jmxProperties; } - protected void addJmxProperties(Map<String, String> jmxProperties, - Map<String, Map<String, Object>> categories, - Map<String, Metric> relevantMetrics) { - for (String metricName : relevantMetrics.keySet()) { - Metric metric = relevantMetrics.get(metricName); - String beanName = metric.getKeyName(); - Object value = categories.get(beanName); - if (value instanceof Map) { - Map<?, ?> map = (Map<?, ?>) value; - for (List<String> matcher : metric.getMatchers()) { - boolean foundMetrics = false; - for (int matchIndex = 0; matchIndex < matcher.size(); matchIndex++) { - String matchKey = matcher.get(matchIndex); - value = map.get(matchKey); - if (value instanceof Map) { - map = (Map<?, ?>) value; - continue; - } else { - if (value != null && matchIndex == matcher.size() - 1) { - jmxProperties.put(metricName, value.toString()); - foundMetrics = true; - } else { - break; - } - } - } - if (foundMetrics) { - break; - } - } - } - } - } - private Map<String, Metric> getRelevantMetrics(Map<String, Map<String, Map<String, Metric>>> metrics, List<String> comps) { Map<String, Metric> relevantMetrics = new HashMap<String, Metric>(); @@ -256,19 +214,6 @@ public class SliderAppMasterClient extends BaseHttpClient { return relevantMetrics; } - private String getCategory(Map<String, Object> bean) { - if (bean.containsKey(NAME_KEY)) { - String name = (String) bean.get(NAME_KEY); - - if (bean.containsKey(PORT_KEY)) { - String port = (String) bean.get(PORT_KEY); - name = name.replace(FORPORT_KEY + port, ""); - } - return name; - } - return null; - } - public static class SliderAppMasterData { public String registryUrl; public String uiUrl; http://git-wip-us.apache.org/repos/asf/ambari/blob/d1c4a93f/contrib/views/slider/src/test/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClientTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/test/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClientTest.java b/contrib/views/slider/src/test/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClientTest.java index 2c0aa1c..a77cdfd 100644 --- a/contrib/views/slider/src/test/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClientTest.java +++ b/contrib/views/slider/src/test/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClientTest.java @@ -21,6 +21,8 @@ package org.apache.ambari.view.slider.rest.client; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,19 +32,19 @@ public class SliderAppMasterClientTest { @Test public void testSliderClientClassAvailability() { SliderAppMasterClient client = new SliderAppMasterClient("http://tmpurl.org";); - Map<String, Metric> relevantMetric = new HashMap<String, Metric>(); + Map<String, Metric> metrics = new HashMap<String, Metric>(); Map<String, String> jmxProperties = new HashMap<String, String>(); Map<String, Map<String, Object>> categories = new HashMap<String, Map<String, Object>>(); - relevantMetric.put("metricAverageLoad", - new Metric("Hadoop:service=HBase,name=Master,sub=Server.averageLoad", true, false)); - relevantMetric.put("DeadRegionServers", - new Metric("Hadoop:service=HBase,name=Master,sub=Server.numDeadRegionServers", true, false)); - relevantMetric.put("ClusterId", - new Metric("Hadoop:service=HBase,name=Master,sub=Server.tag.clusterId", true, false)); - relevantMetric.put("IsActiveMaster", - new Metric("Hadoop:service=HBase,name=Master,sub=Server.tag.isActiveMaster", true, false)); - relevantMetric.put("peakUsageCommitted", - new Metric("java#lang:type=MemoryPool,name=Code Cache.PeakUsage.committed", true, false)); + metrics.put("metricAverageLoad", + new Metric("Hadoop:service=HBase,name=Master,sub=Server.averageLoad", true, false)); + metrics.put("DeadRegionServers", + new Metric("Hadoop:service=HBase,name=Master,sub=Server.numDeadRegionServers", true, false)); + metrics.put("ClusterId", + new Metric("Hadoop:service=HBase,name=Master,sub=Server.tag.clusterId", true, false)); + metrics.put("IsActiveMaster", + new Metric("Hadoop:service=HBase,name=Master,sub=Server.tag.isActiveMaster", true, false)); + metrics.put("peakUsageCommitted", + new Metric("java#lang:type=MemoryPool,name=Code Cache.PeakUsage.committed", true, false)); Map<String, Object> masterServer = new HashMap<String, Object>(); masterServer.put("averageLoad", "0.1"); @@ -57,21 +59,21 @@ public class SliderAppMasterClientTest { memPool.put("SomeOther", "other"); categories.put("java.lang:type=MemoryPool,name=Code Cache", memPool); - client.addJmxProperties(jmxProperties, categories, relevantMetric); + SliderAppJmxHelper.addJmxPropertiesFromBeans(jmxProperties, categories, metrics); Assert.assertEquals(jmxProperties.size(), 4); } @Test public void testMetricMatchers() throws Exception { Metric m1 = new Metric("a_b.c", true, false); - Assert.assertEquals(m1.getKeyName(), "a_b"); + Assert.assertEquals(m1.getJmxBeanKeyName(), "a_b"); List<List<String>> matchers = m1.getMatchers(); Assert.assertEquals(matchers.size(), 1); Assert.assertEquals(matchers.get(0).size(), 1); Assert.assertEquals(matchers.get(0).get(0), "c"); m1 = new Metric("a_b.c.d", true, false); - Assert.assertEquals(m1.getKeyName(), "a_b"); + Assert.assertEquals(m1.getJmxBeanKeyName(), "a_b"); matchers = m1.getMatchers(); Assert.assertEquals(matchers.size(), 2); Assert.assertEquals(matchers.get(0).size(), 2); @@ -81,7 +83,7 @@ public class SliderAppMasterClientTest { Assert.assertEquals(matchers.get(1).get(0), "c.d"); m1 = new Metric("a_b.c.d.e", true, false); - Assert.assertEquals(m1.getKeyName(), "a_b"); + Assert.assertEquals(m1.getJmxBeanKeyName(), "a_b"); matchers = m1.getMatchers(); Assert.assertEquals(matchers.size(), 4); Assert.assertEquals(matchers.get(0).size(), 3); @@ -93,4 +95,83 @@ public class SliderAppMasterClientTest { Assert.assertEquals(matchers.get(3).size(), 1); Assert.assertEquals(matchers.get(3).get(0), "c.d.e"); } + + @Test + public void testReadMetricsFromJson() throws Exception { + String jsonJmx = "{\n" + + "\"tasks.total\": 28,\n" + + "\"slots.total\": 1,\n" + + "\"slots.free\": 0,\n" + + "\"supervisors\": 1,\n" + + "\"executors.total\": 28,\n" + + "\"slots.used\": 1,\n" + + "\"topologies\": 1,\n" + + "\"nimbus.uptime\": 2026\n" + + "}"; + InputStream stream = new ByteArrayInputStream(jsonJmx.getBytes("UTF-8")); + Map<String, Metric> metrics = new HashMap<String, Metric>(); + Map<String, String> jmxProperties = new HashMap<String, String>(); + metrics.put("FreeSlots", new Metric("$['slots.free']", true, false)); + metrics.put("Tasks", new Metric("$['tasks.total']", true, false)); + metrics.put("Executors", new Metric("$['executors.total']", true, false)); + metrics.put("Items", new Metric("['items']", true, false)); + SliderAppJmxHelper.extractMetricsFromJmxJson(stream, "jmxurl", jmxProperties, metrics); + Assert.assertEquals(jmxProperties.size(), 3); + Assert.assertEquals(jmxProperties.get("FreeSlots"), "0"); + Assert.assertEquals(jmxProperties.get("Tasks"), "28"); + Assert.assertEquals(jmxProperties.get("Executors"), "28"); + } + + @Test + public void testReadMetricsFromXml() throws Exception { + String jsonJmx = "<stats>\n" + + "<masterGoalState>NORMAL</masterGoalState>\n" + + "<masterState>NORMAL</masterState>\n" + + "<badTabletServers></badTabletServers>\n" + + "<tabletServersShuttingDown></tabletServersShuttingDown>\n" + + "<unassignedTablets>0</unassignedTablets>\n" + + "<deadTabletServers></deadTabletServers>\n" + + "<deadLoggers></deadLoggers>\n" + + "<tables>\n" + + "<table>\n" + + "<tablename>!METADATA</tablename>\n" + + "<tableId>!0</tableId>\n" + + "<tableState>ONLINE</tableState>\n" + + "<tablets>3</tablets>\n" + + "<onlineTablets>3</onlineTablets>\n" + + "<recs>49</recs>\n" + + "<recsInMemory>24</recsInMemory>\n" + + "<ingest>1.1271868150075986E-4</ingest>\n" + + "<ingestByteRate>0.00475332746606865</ingestByteRate>\n" + + "<query>0.014071304698085596</query>\n" + + "<queryByteRate>0.014071304698085596</queryByteRate>\n" + + "<majorCompactions>\n" + + "<running>0</running>\n" + + "<queued>0</queued>\n" + + "</majorCompactions>\n" + + "</table>\n" + + "</tables>\n" + + "<totals>\n" + + "<ingestrate>0.0016737652847869573</ingestrate>\n" + + "<queryrate>0.014071304698085596</queryrate>\n" + + "<diskrate>0.0</diskrate>\n" + + "<numentries>554</numentries>\n" + + "</totals>\n" + + "</stats>"; + InputStream stream = new ByteArrayInputStream(jsonJmx.getBytes("UTF-8")); + Map<String, Metric> metrics = new HashMap<String, Metric>(); + Map<String, String> jmxProperties = new HashMap<String, String>(); + metrics.put("masterGoalState", new Metric("/stats/masterGoalState", true, false)); + metrics.put("masterState", new Metric("/stats/masterState", true, false)); + metrics.put("totals_diskrate", new Metric("/stats/totals/diskrate", true, false)); + metrics.put("totals_diskaccess", new Metric("/stats/totals/diskaccess", true, false)); + metrics.put("badTabletServers", new Metric("/stats/badTabletServers", true, false)); + SliderAppJmxHelper.extractMetricsFromJmxXML(stream, "jmxurl", jmxProperties, metrics); + Assert.assertEquals(jmxProperties.size(), 5); + Assert.assertEquals(jmxProperties.get("masterGoalState"), "NORMAL"); + Assert.assertEquals(jmxProperties.get("masterState"), "NORMAL"); + Assert.assertEquals(jmxProperties.get("totals_diskrate"), "0.0"); + Assert.assertEquals(jmxProperties.get("totals_diskaccess"), ""); + Assert.assertEquals(jmxProperties.get("badTabletServers"), ""); + } }
