Repository: ambari Updated Branches: refs/heads/trunk b911c05e1 -> da413e507
AMBARI-4871. All JMX metrics are not working on the UI - the metrics are not showing up. (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/da413e50 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/da413e50 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/da413e50 Branch: refs/heads/trunk Commit: da413e507e7045012dfe995622456459008e87e1 Parents: b911c05 Author: Lisnichenko Dmitro <[email protected]> Authored: Thu Feb 27 17:25:55 2014 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Thu Feb 27 19:56:47 2014 +0200 ---------------------------------------------------------------------- .../controller/jmx/JMXPropertyProvider.java | 207 ++++++++++++------- .../HDP/2.1.1/services/STORM/metrics.json | 97 +++++++++ .../AmbariManagementControllerTest.java | 2 +- .../StackDefinedPropertyProviderTest.java | 49 ++++- .../controller/jmx/JMXPropertyProviderTest.java | 2 + .../controller/jmx/TestStreamProvider.java | 1 + .../resources/stacks/HDP/2.1.1/metainfo.xml | 24 +++ .../stacks/HDP/2.1.1/repos/repoinfo.xml | 61 ++++++ .../stacks/HDP/2.1.1/role_command_order.json | 100 +++++++++ .../STORM/configuration/placeholder.txt | 17 ++ .../HDP/2.1.1/services/STORM/metainfo.xml | 112 ++++++++++ .../HDP/2.1.1/services/STORM/metrics.json | 99 +++++++++ .../services/STORM/package/placeholder.txt | 17 ++ .../src/test/resources/storm_rest_api_jmx.json | 10 + 14 files changed, 719 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java index 4de36ca..927eb00 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java @@ -40,16 +40,17 @@ import java.util.regex.Pattern; import org.apache.ambari.server.controller.internal.AbstractPropertyProvider; import org.apache.ambari.server.controller.internal.PropertyInfo; -import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SystemException; -import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; +import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.apache.ambari.server.controller.utilities.StreamProvider; +import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectReader; +import org.codehaus.jackson.type.TypeReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,6 +65,7 @@ public class JMXPropertyProvider extends AbstractPropertyProvider { private static final long DEFAULT_POPULATE_TIMEOUT_MILLIS = 10000L; public static final String TIMED_OUT_MSG = "Timed out waiting for JMX metrics."; + public static final String STORM_REST_API = "STORM_REST_API"; /** * Thread pool @@ -89,7 +91,8 @@ public class JMXPropertyProvider extends AbstractPropertyProvider { EXECUTOR_SERVICE = threadPoolExecutor; } - private final static ObjectReader objectReader; + private final static ObjectReader jmxObjectReader; + private final static ObjectReader stormObjectReader; private static final Map<String, String> DEFAULT_JMX_PORTS = new HashMap<String, String>(); @@ -104,10 +107,19 @@ public class JMXPropertyProvider extends AbstractPropertyProvider { DEFAULT_JMX_PORTS.put("HISTORYSERVER", "19888"); DEFAULT_JMX_PORTS.put("NODEMANAGER", "8042"); DEFAULT_JMX_PORTS.put("JOURNALNODE", "8480"); - - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationConfig.Feature.USE_ANNOTATIONS, false); - objectReader = objectMapper.reader(JMXMetricHolder.class); + DEFAULT_JMX_PORTS.put("STORM_REST_API", "8745"); + + ObjectMapper jmxObjectMapper = new ObjectMapper(); + jmxObjectMapper.configure(DeserializationConfig.Feature.USE_ANNOTATIONS, false); + jmxObjectReader = jmxObjectMapper.reader(JMXMetricHolder.class); + + JsonFactory factory = new JsonFactory(); + ObjectMapper stormObjectMapper = new ObjectMapper(factory); + TypeReference<HashMap<String,Object>> typeRef + = new TypeReference< + HashMap<String,Object> + >() {}; + stormObjectReader = jmxObjectMapper.reader(typeRef); } protected final static Logger LOG = @@ -232,8 +244,12 @@ public class JMXPropertyProvider extends AbstractPropertyProvider { * * @return the spec */ - protected String getSpec(String protocol, String hostName, String port) { - return protocol + "://" + hostName + ":" + port + "/jmx"; + protected String getSpec(String protocol, String hostName, + String port, String componentName) { + if (null == componentName || !componentName.equals(STORM_REST_API)) + return protocol + "://" + hostName + ":" + port + "/jmx"; + else + return protocol + "://" + hostName + ":" + port + "/api/cluster/summary"; } /** @@ -245,7 +261,7 @@ public class JMXPropertyProvider extends AbstractPropertyProvider { * @return the spec */ protected String getSpec(String hostName, String port) { - return getSpec("http", hostName, port); + return getSpec("http", hostName, port, null); } /** @@ -324,94 +340,131 @@ public class JMXPropertyProvider extends AbstractPropertyProvider { String protocol = getJMXProtocol(clusterName, componentName); try { - InputStream in = streamProvider.readFrom(getSpec(protocol, hostName, port)); + InputStream in = streamProvider.readFrom(getSpec(protocol, hostName, port, componentName)); try { - JMXMetricHolder metricHolder = objectReader.readValue(in); - - Map<String, Map<String, Object>> categories = new HashMap<String, Map<String, Object>>(); - for (Map<String, Object> bean : metricHolder.getBeans()) { - String category = getCategory(bean); - if (category != null) { - categories.put(category, bean); - } + if (null == componentName || !componentName.equals(STORM_REST_API)) { + getHadoopMetricValue(in, ids, resource, request); + } else { + getStormMetricValue(in, ids, resource, request); } - for (String propertyId : ids) { - Map<String, PropertyInfo> propertyInfoMap = getPropertyInfoMap(componentName, propertyId); + } finally { + in.close(); + } + } catch (IOException e) { + logException(e); + } + return resource; + } + + /** + * Hadoop-specific metrics fetching + */ + private void getHadoopMetricValue(InputStream in, Set<String> ids, + Resource resource, Request request) throws IOException { + JMXMetricHolder metricHolder = jmxObjectReader.readValue(in); + + Map<String, Map<String, Object>> categories = new HashMap<String, Map<String, Object>>(); + String componentName = (String) resource.getPropertyValue(componentNamePropertyId); + + for (Map<String, Object> bean : metricHolder.getBeans()) { + String category = getCategory(bean); + if (category != null) { + categories.put(category, bean); + } + } - String requestedPropertyId = propertyId; + for (String propertyId : ids) { + Map<String, PropertyInfo> propertyInfoMap = getPropertyInfoMap(componentName, propertyId); - for (Map.Entry<String, PropertyInfo> entry : propertyInfoMap.entrySet()) { + String requestedPropertyId = propertyId; - PropertyInfo propertyInfo = entry.getValue(); - propertyId = entry.getKey(); + for (Map.Entry<String, PropertyInfo> entry : propertyInfoMap.entrySet()) { - if (propertyInfo.isPointInTime()) { + PropertyInfo propertyInfo = entry.getValue(); + propertyId = entry.getKey(); - String property = propertyInfo.getPropertyId(); - String category = ""; + if (propertyInfo.isPointInTime()) { - - List<String> keyList = new LinkedList<String>(); - - int keyStartIndex = property.indexOf('['); - if (-1 != keyStartIndex) { - int keyEndIndex = property.indexOf(']', keyStartIndex); - if (-1 != keyEndIndex && keyEndIndex > keyStartIndex) { - keyList.add(property.substring(keyStartIndex+1, keyEndIndex)); - } - } - - if (!containsArguments(propertyId)) { - int dotIndex = property.indexOf('.', property.indexOf('=')); - if (-1 != dotIndex) { - category = property.substring(0, dotIndex); - property = (-1 == keyStartIndex) ? + String property = propertyInfo.getPropertyId(); + String category = ""; + + + List<String> keyList = new LinkedList<String>(); + + int keyStartIndex = property.indexOf('['); + if (-1 != keyStartIndex) { + int keyEndIndex = property.indexOf(']', keyStartIndex); + if (-1 != keyEndIndex && keyEndIndex > keyStartIndex) { + keyList.add(property.substring(keyStartIndex+1, keyEndIndex)); + } + } + + if (!containsArguments(propertyId)) { + int dotIndex = property.indexOf('.', property.indexOf('=')); + if (-1 != dotIndex) { + category = property.substring(0, dotIndex); + property = (-1 == keyStartIndex) ? property.substring(dotIndex+1) : - property.substring(dotIndex+1, keyStartIndex); - } - } else { - int firstKeyIndex = keyStartIndex > -1 ? keyStartIndex : property.length(); - int dotIndex = property.lastIndexOf('.', firstKeyIndex); + property.substring(dotIndex+1, keyStartIndex); + } + } else { + int firstKeyIndex = keyStartIndex > -1 ? keyStartIndex : property.length(); + int dotIndex = property.lastIndexOf('.', firstKeyIndex); - if (dotIndex != -1) { - category = property.substring(0, dotIndex); - property = property.substring(dotIndex + 1, firstKeyIndex); - } - } + if (dotIndex != -1) { + category = property.substring(0, dotIndex); + property = property.substring(dotIndex + 1, firstKeyIndex); + } + } + + if (containsArguments(propertyId)) { + Pattern pattern = Pattern.compile(category); - if (containsArguments(propertyId)) { - Pattern pattern = Pattern.compile(category); - - // find all jmx categories that match the regex - for (String jmxCat : categories.keySet()) { - Matcher matcher = pattern.matcher(jmxCat); - if (matcher.matches()) { - String newPropertyId = propertyId; - for (int i = 0; i < matcher.groupCount(); i++) { - newPropertyId = substituteArgument(newPropertyId, "$" + (i + 1), matcher.group(i + 1)); - } - // We need to do the final filtering here, after the argument substitution - if (isRequestedPropertyId(newPropertyId, requestedPropertyId, request)) { - setResourceValue(resource, categories, newPropertyId, jmxCat, property, keyList); - } - } + // find all jmx categories that match the regex + for (String jmxCat : categories.keySet()) { + Matcher matcher = pattern.matcher(jmxCat); + if (matcher.matches()) { + String newPropertyId = propertyId; + for (int i = 0; i < matcher.groupCount(); i++) { + newPropertyId = substituteArgument(newPropertyId, "$" + (i + 1), matcher.group(i + 1)); + } + // We need to do the final filtering here, after the argument substitution + if (isRequestedPropertyId(newPropertyId, requestedPropertyId, request)) { + setResourceValue(resource, categories, newPropertyId, jmxCat, property, keyList); } - } else { - setResourceValue(resource, categories, propertyId, category, property, keyList); } } + } else { + setResourceValue(resource, categories, propertyId, category, property, keyList); } } - } finally { - in.close(); } - } catch (IOException e) { - logException(e); } - return resource; + } + + /** + * TODO: Refactor + * Storm-specific metrics fetching + */ + private void getStormMetricValue(InputStream in, Set<String> ids, + Resource resource, Request request) throws IOException { + HashMap<String, Object> metricHolder = stormObjectReader.readValue(in); + for (String category : ids) { + Map<String, PropertyInfo> defProps = getComponentMetrics().get(STORM_REST_API); + for (String depProp : defProps.keySet()) { + if (depProp.startsWith(category)) { + PropertyInfo propInfo = defProps.get(depProp); + String propName = propInfo.getPropertyId(); + Object propertyValue = metricHolder.get(propName); + String absId = PropertyHelper.getPropertyId(category, propName); + // TODO: Maybe cast to int + resource.setProperty(absId, propertyValue); + } + } + } } private void setResourceValue(Resource resource, Map<String, Map<String, Object>> categories, String propertyId, http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metrics.json b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metrics.json index da75ef8..ef3cde7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metrics.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metrics.json @@ -1,4 +1,101 @@ { + "STORM_REST_API": { + "Component": [ + { + "type": "jmx", + "metrics": { + "metrics/api/cluster/summary/tasks.total": { + "metric": "tasks.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.total": { + "metric": "slots.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.free": { + "metric": "slots.free", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/supervisors": { + "metric": "supervisors", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/executors.total": { + "metric": "executors.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.used": { + "metric": "slots.used", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/topologies": { + "metric": "topologies", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/nimbus.uptime": { + "metric": "nimbus.uptime", + "pointInTime": true, + "temporal": false + } + } + } + ], + "HostComponent": [ + { + "type": "jmx", + "metrics": { + "metrics/api/cluster/summary/tasks.total": { + "metric": "tasks.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.total": { + "metric": "slots.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.free": { + "metric": "slots.free", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/supervisors": { + "metric": "supervisors", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/executors.total": { + "metric": "executors.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.used": { + "metric": "slots.used", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/topologies": { + "metric": "topologies", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/nimbus.uptime": { + "metric": "nimbus.uptime", + "pointInTime": true, + "temporal": false + } + } + } + + ] + }, "NIMBUS": { "Component": [ { http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index dc4c5a5..6e32ac8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -147,7 +147,7 @@ public class AmbariManagementControllerTest { private static final String REPO_ID = "HDP-1.1.1.16"; private static final String PROPERTY_NAME = "hbase.regionserver.msginterval"; private static final String SERVICE_NAME = "HDFS"; - private static final int STACK_VERSIONS_CNT = 11; + private static final int STACK_VERSIONS_CNT = 12; private static final int REPOS_CNT = 3; private static final int STACKS_CNT = 1; private static final int STACK_PROPERTIES_CNT = 81; http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java index 6ce0022..b9b2962 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java @@ -656,7 +656,54 @@ public class StackDefinedPropertyProviderTest { Assert.assertEquals(2, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/dfs/journalnode/cluster/mycluster", "lastWriterEpoch"))); Assert.assertEquals(0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/dfs/journalnode/cluster/mycluster", "currentLagTxns"))); Assert.assertEquals(8444, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/dfs/journalnode/cluster/mycluster", "lastWrittenTxId"))); - } + } + + + + @Test + public void testPopulateResources_jmx_Storm() throws Exception { + // Adjust stack version for cluster + Cluster cluster = clusters.getCluster("c1"); + cluster.setDesiredStackVersion(new StackId("HDP-2.1.1")); + + TestStreamProvider streamProvider = new TestStreamProvider(); + TestJMXHostProvider hostProvider = new TestJMXHostProvider(false); + TestGangliaHostProvider gangliaHostProvider = new TestGangliaHostProvider(); + + StackDefinedPropertyProvider propertyProvider = new StackDefinedPropertyProvider( + Resource.Type.HostComponent, + hostProvider, + gangliaHostProvider, + streamProvider, + PropertyHelper.getPropertyId("HostRoles", "cluster_name"), + PropertyHelper.getPropertyId("HostRoles", "host_name"), + PropertyHelper.getPropertyId("HostRoles", "component_name"), + PropertyHelper.getPropertyId("HostRoles", "state"), + new EmptyPropertyProvider(), + new EmptyPropertyProvider()); + + Resource resource = new ResourceImpl(Resource.Type.HostComponent); + + resource.setProperty("HostRoles/cluster_name", "c1"); + resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-0e-34-e1.compute-1.internal"); + resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "STORM_REST_API"); + resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "STARTED"); + + // request with an empty set should get all supported properties + Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet()); + + Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); + + // see test/resources/storm_rest_api_jmx.json for values + Assert.assertEquals(28, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/tasks.total", "tasks.total"))); + Assert.assertEquals(8, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/slots.total", "slots.total"))); + Assert.assertEquals(5, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/slots.free", "slots.free"))); + Assert.assertEquals(2, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/supervisors", "supervisors"))); + Assert.assertEquals(28, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/executors.total", "executors.total"))); + Assert.assertEquals(3, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/slots.used", "slots.used"))); + Assert.assertEquals(1, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/topologies", "topologies"))); + Assert.assertEquals(4637, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/api/cluster/summary/nimbus.uptime", "nimbus.uptime"))); + } @Test public void testPopulateResources_NoRegionServer() throws Exception { http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java index c2910db..ac7314e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java @@ -450,6 +450,8 @@ public class JMXPropertyProviderTest { return null == clusterName ? "60010" : "60011"; else if (componentName.equals("JOURNALNODE")) return "8480"; + else if (componentName.equals("STORM_REST_API")) + return "8745"; else return null; } http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java index 0c793dc..a7b7553 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java @@ -38,6 +38,7 @@ public class TestStreamProvider implements StreamProvider { FILE_MAPPING.put("60011", "hbase_hbasemaster_jmx_2.json"); FILE_MAPPING.put("8088", "resourcemanager_jmx.json"); FILE_MAPPING.put("8480", "hdfs_journalnode_jmx.json"); + FILE_MAPPING.put("8745", "storm_rest_api_jmx.json"); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/stacks/HDP/2.1.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/metainfo.xml new file mode 100644 index 0000000..04416b0 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/metainfo.xml @@ -0,0 +1,24 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <versions> + <active>true</active> + </versions> + <extends>2.0.8</extends> +</metainfo> + http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/stacks/HDP/2.1.1/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/repos/repoinfo.xml new file mode 100644 index 0000000..1025240 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/repos/repoinfo.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os type="centos6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0</baseurl> + <repoid>HDP-2.0.8</repoid> + <reponame>HDP</reponame> + </repo> + </os> + <os type="centos5"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos5/2.x/updates/2.0.6.0</baseurl> + <repoid>HDP-2.0.8</repoid> + <reponame>HDP</reponame> + </repo> + </os> + <os type="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0</baseurl> + <repoid>HDP-2.0.8</repoid> + <reponame>HDP</reponame> + </repo> + </os> + <os type="redhat5"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos5/2.x/updates/2.0.6.0</baseurl> + <repoid>HDP-2.0.8</repoid> + <reponame>HDP</reponame> + </repo> + </os> + <os type="suse11"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11/2.x/updates/2.0.6.0</baseurl> + <repoid>HDP-2.0.8</repoid> + <reponame>HDP</reponame> + </repo> + </os> + <os type="sles11"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11/2.x/updates/2.0.6.0</baseurl> + <repoid>HDP-2.0.8</repoid> + <reponame>HDP</reponame> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/stacks/HDP/2.1.1/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/role_command_order.json b/ambari-server/src/test/resources/stacks/HDP/2.1.1/role_command_order.json new file mode 100644 index 0000000..84610ca --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/role_command_order.json @@ -0,0 +1,100 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "NAGIOS_SERVER-INSTALL" : ["HIVE_CLIENT-INSTALL", "HCAT-INSTALL", + "MAPREDUCE_CLIENT-INSTALL", "OOZIE_CLIENT-INSTALL"], + "HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START"], + "HBASE_REGIONSERVER-START": ["HBASE_MASTER-START"], + "OOZIE_SERVER-START": ["JOBTRACKER-START", "TASKTRACKER-START"], + "WEBHCAT_SERVER-START": ["TASKTRACKER-START", "HIVE_SERVER-START"], + "HIVE_METASTORE-START": ["MYSQL_SERVER-START"], + "HIVE_SERVER-START": ["TASKTRACKER-START", "MYSQL_SERVER-START"], + "HUE_SERVER-START": ["HIVE_SERVER-START", "HCAT-START", "OOZIE_SERVER-START"], + "FLUME_SERVER-START": ["OOZIE_SERVER-START"], + "NAGIOS_SERVER-START": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START", + "GANGLIA_SERVER-START", "GANGLIA_MONITOR-START", "HCAT-START", + "HIVE_SERVER-START", "HIVE_METASTORE-START", "HUE_SERVER-START", + "JOBTRACKER-START", "TASKTRACKER-START", "ZOOKEEPER_SERVER-START", + "MYSQL_SERVER-START", "OOZIE_SERVER-START", "PIG-START", "SQOOP-START", + "WEBHCAT_SERVER-START", "FLUME_SERVER-START"], + "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], + "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START"], + "WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"], + "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], + "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"], + "HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"], + "PIG_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], + "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], + "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], + "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], + "ZOOKEEPER_SERVER-STOP" : ["HBASE_MASTER-STOP", "HBASE_REGIONSERVER-STOP"], + "HBASE_MASTER-STOP": ["HBASE_REGIONSERVER-STOP"], + "TASKTRACKER-UPGRADE": ["JOBTRACKER-UPGRADE"], + "MAPREDUCE_CLIENT-UPGRADE": ["TASKTRACKER-UPGRADE", "JOBTRACKER-UPGRADE"], + "ZOOKEEPER_SERVER-UPGRADE": ["MAPREDUCE_CLIENT-UPGRADE"], + "ZOOKEEPER_CLIENT-UPGRADE": ["ZOOKEEPER_SERVER-UPGRADE"], + "HBASE_MASTER-UPGRADE": ["ZOOKEEPER_CLIENT-UPGRADE"], + "HBASE_REGIONSERVER-UPGRADE": ["HBASE_MASTER-UPGRADE"], + "HBASE_CLIENT-UPGRADE": ["HBASE_REGIONSERVER-UPGRADE"], + "HIVE_SERVER-UPGRADE" : ["HBASE_CLIENT-UPGRADE"], + "HIVE_METASTORE-UPGRADE" : ["HIVE_SERVER-UPGRADE"], + "MYSQL_SERVER-UPGRADE": ["HIVE_METASTORE-UPGRADE"], + "HIVE_CLIENT-UPGRADE": ["MYSQL_SERVER-UPGRADE"], + "HCAT-UPGRADE": ["HIVE_CLIENT-UPGRADE"], + "OOZIE_SERVER-UPGRADE" : ["HCAT-UPGRADE"], + "OOZIE_CLIENT-UPGRADE" : ["OOZIE_SERVER-UPGRADE"], + "WEBHCAT_SERVER-UPGRADE" : ["OOZIE_CLIENT-UPGRADE"], + "PIG-UPGRADE" : ["WEBHCAT_SERVER-UPGRADE"], + "SQOOP-UPGRADE" : ["PIG-UPGRADE"], + "NAGIOS_SERVER-UPGRADE" : ["SQOOP-UPGRADE"], + "GANGLIA_SERVER-UPGRADE" : ["NAGIOS_SERVER-UPGRADE"], + "GANGLIA_MONITOR-UPGRADE" : ["GANGLIA_SERVER-UPGRADE"] + }, + "_comment" : "GLUSTERFS-specific dependencies", + "optional_glusterfs": { + "HBASE_MASTER-START": ["PEERSTATUS-START"], + "JOBTRACKER-START": ["PEERSTATUS-START"], + "TASKTRACKER-START": ["PEERSTATUS-START"], + "GLUSTERFS_SERVICE_CHECK-SERVICE_CHECK": ["PEERSTATUS-START"], + "JOBTRACKER-UPGRADE": ["GLUSTERFS_CLIENT-UPGRADE"] + }, + "_comment" : "Dependencies that are used when GLUSTERFS is not present in cluster", + "optional_no_glusterfs": { + "SECONDARY_NAMENODE-START": ["NAMENODE-START"], + "RESOURCEMANAGER-START": ["NAMENODE-START", "DATANODE-START"], + "NODEMANAGER-START": ["NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START"], + "HISTORYSERVER-START": ["NAMENODE-START", "DATANODE-START"], + "HBASE_MASTER-START": ["NAMENODE-START", "DATANODE-START"], + "JOBTRACKER-START": ["NAMENODE-START", "DATANODE-START"], + "TASKTRACKER-START": ["NAMENODE-START", "DATANODE-START"], + "HIVE_SERVER-START": ["DATANODE-START"], + "WEBHCAT_SERVER-START": ["DATANODE-START"], + "NAGIOS_SERVER-START": ["NAMENODE-START", "SECONDARY_NAMENODE-START", + "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START", "HISTORYSERVER-START"], + "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["NAMENODE-START", "DATANODE-START", + "SECONDARY_NAMENODE-START"], + "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", + "RESOURCEMANAGER-START", "HISTORYSERVER-START", "YARN_SERVICE_CHECK-SERVICE_CHECK"], + "YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"], + "PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"], + "NAMENODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP", + "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], + "DATANODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP", + "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], + "SECONDARY_NAMENODE-UPGRADE": ["NAMENODE-UPGRADE"], + "DATANODE-UPGRADE": ["SECONDARY_NAMENODE-UPGRADE"], + "HDFS_CLIENT-UPGRADE": ["DATANODE-UPGRADE"], + "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"] + }, + "_comment" : "Dependencies that are used in HA NameNode cluster", + "optional_ha": { + "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"], + "ZKFC-START": ["NAMENODE-START"], + "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"], + "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"] + } +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/configuration/placeholder.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/configuration/placeholder.txt b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/configuration/placeholder.txt new file mode 100644 index 0000000..e2479f3 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/configuration/placeholder.txt @@ -0,0 +1,17 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml new file mode 100644 index 0000000..f2bceca --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml @@ -0,0 +1,112 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>STORM</name> + <comment>Apache Hadoop Stream processing framework</comment> + <version>0.9.0.1</version> + <components> + + <component> + <name>NIMBUS</name> + <category>MASTER</category> + <commandScript> + <script>scripts/nimbus.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>STORM_REST_API</name> + <category>MASTER</category> + <commandScript> + <script>scripts/rest_api.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>SUPERVISOR</name> + <category>SLAVE</category> + <commandScript> + <script>scripts/supervisor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>STORM_UI_SERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/ui_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>DRPC_SERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/drpc_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>LOGVIEWER_SERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/logviewer_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>rpm</type> + <name>storm</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <configuration-dependencies> + </configuration-dependencies> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json new file mode 100644 index 0000000..59bec39 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metrics.json @@ -0,0 +1,99 @@ +{ + "STORM_REST_API": { + "Component": [ + { + "type": "jmx", + "metrics": { + "metrics/api/cluster/summary/tasks.total": { + "metric": "tasks.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.total": { + "metric": "slots.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.free": { + "metric": "slots.free", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/supervisors": { + "metric": "supervisors", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/executors.total": { + "metric": "executors.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.used": { + "metric": "slots.used", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/topologies": { + "metric": "topologies", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/nimbus.uptime": { + "metric": "nimbus.uptime", + "pointInTime": true, + "temporal": false + } + } + } + ], + "HostComponent": [ + { + "type": "jmx", + "metrics": { + "metrics/api/cluster/summary/tasks.total": { + "metric": "tasks.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.total": { + "metric": "slots.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.free": { + "metric": "slots.free", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/supervisors": { + "metric": "supervisors", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/executors.total": { + "metric": "executors.total", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/slots.used": { + "metric": "slots.used", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/topologies": { + "metric": "topologies", + "pointInTime": true, + "temporal": false + }, + "metrics/api/cluster/summary/nimbus.uptime": { + "metric": "nimbus.uptime", + "pointInTime": true, + "temporal": false + } + } + } + + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/package/placeholder.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/package/placeholder.txt b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/package/placeholder.txt new file mode 100644 index 0000000..e2479f3 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/package/placeholder.txt @@ -0,0 +1,17 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent http://git-wip-us.apache.org/repos/asf/ambari/blob/da413e50/ambari-server/src/test/resources/storm_rest_api_jmx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/storm_rest_api_jmx.json b/ambari-server/src/test/resources/storm_rest_api_jmx.json new file mode 100644 index 0000000..d099d13 --- /dev/null +++ b/ambari-server/src/test/resources/storm_rest_api_jmx.json @@ -0,0 +1,10 @@ +{ + "tasks.total" : 28, + "slots.total" : 8, + "slots.free" : 5, + "supervisors" : 2, + "executors.total" : 28, + "slots.used" : 3, + "topologies" : 1, + "nimbus.uptime" : 4637 +}
