Repository: ambari Updated Branches: refs/heads/branch-2.4 f516c79e1 -> 7d9818ba8 refs/heads/trunk 32890b554 -> 35ff7d79a
AMBARI-17063. Retrieve specific metrics when Ambari queries NameNode HA states (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/35ff7d79 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/35ff7d79 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/35ff7d79 Branch: refs/heads/trunk Commit: 35ff7d79a8b8c2c74bf772efff295af1dc4091f8 Parents: 32890b5 Author: Andrew Onishuk <[email protected]> Authored: Wed Jun 8 13:40:40 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Wed Jun 8 13:40:40 2016 +0300 ---------------------------------------------------------------------- .../controller/jmx/JMXPropertyProvider.java | 48 ++++++++- .../controller/jmx/TestStreamProvider.java | 19 +++- .../metrics/JMXPropertyProviderTest.java | 102 ++++++++++++++++++- .../resources/hdfs_namenode_jmx_ha_only.json | 7 ++ 4 files changed, 167 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/35ff7d79/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java index a315e5c..7665d7f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java @@ -86,6 +86,19 @@ public class JMXPropertyProvider extends ThreadPoolEnabledPropertyProvider { private static final Map<String, String> DEFAULT_JMX_PORTS = new HashMap<String, String>(); + /** + * When Ambari queries NameNode's HA state (among other metrics), it retrieves all metrics from "NN_URL:port/jmx". + * But some metrics may compete for the NameNode lock and a request to /jmx may take much time. + * <p> + * The properties from this map will be retrieved using a provided URL query. + * Even if JMX is locked and a request for all metrics is waiting (/jmx is unavailable), + * HAState will be updated via a separate JMX call. + * <p> + * Currently org.apache.hadoop.jmx.JMXJsonServlet can provide only one property per a request, + * each property from this list adds a request to JMX. + */ + private static final Map<String, Map<String, String>> AD_HOC_PROPERTIES = new HashMap<>(); + static { DEFAULT_JMX_PORTS.put("NAMENODE", "50070"); DEFAULT_JMX_PORTS.put("DATANODE", "50075"); @@ -96,6 +109,10 @@ public class JMXPropertyProvider extends ThreadPoolEnabledPropertyProvider { DEFAULT_JMX_PORTS.put("NODEMANAGER", "8042"); DEFAULT_JMX_PORTS.put("JOURNALNODE", "8480"); DEFAULT_JMX_PORTS.put("STORM_REST_API", "8745"); + + AD_HOC_PROPERTIES.put("NAMENODE", + Collections.singletonMap("metrics/dfs/FSNamesystem/HAState", + "/jmx?get=Hadoop:service=NameNode,name=FSNamesystem::tag.HAState")); } protected final static Logger LOG = @@ -249,17 +266,40 @@ public class JMXPropertyProvider extends ThreadPoolEnabledPropertyProvider { // check to see if there is a cached value and use it if there is JMXMetricHolder jmxMetricHolder = metricsRetrievalService.getCachedJMXMetric(jmxUrl); - if (null == jmxMetricHolder) { - return resource; - } // if the ticket becomes invalid (timeout) then bail out if (!ticket.isValid()) { return resource; } - getHadoopMetricValue(jmxMetricHolder, ids, resource, request, ticket); + if (null != jmxMetricHolder) { + getHadoopMetricValue(jmxMetricHolder, ids, resource, request, ticket); + } + if (AD_HOC_PROPERTIES.containsKey(componentName)) { + for (String propertyId : ids) { + for (String adHocId : AD_HOC_PROPERTIES.get(componentName).keySet()) { + String queryURL = null; + // if all metrics from "metrics/dfs/FSNamesystem" were requested, retrieves HAState. + if (adHocId.equals(propertyId) || adHocId.startsWith(propertyId + '/')) { + queryURL = AD_HOC_PROPERTIES.get(componentName).get(adHocId); + } + if (queryURL != null) { + String adHocUrl = getSpec(protocol, hostName, port, queryURL); + metricsRetrievalService.submitJMXRequest(streamProvider, adHocUrl); + JMXMetricHolder adHocJMXMetricHolder = metricsRetrievalService.getCachedJMXMetric(adHocUrl); + + // if the ticket becomes invalid (timeout) then bail out + if (!ticket.isValid()) { + return resource; + } + if (null != adHocJMXMetricHolder) { + getHadoopMetricValue(adHocJMXMetricHolder, Collections.singleton(propertyId), resource, request, ticket); + } + } + } + } + } } catch (IOException e) { AmbariException detailedException = new AmbariException(String.format( "Unable to get JMX metrics from the host %s for the component %s. Spec: %s", hostName, http://git-wip-us.apache.org/repos/asf/ambari/blob/35ff7d79/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java index 8819991..f29dc6f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java @@ -24,7 +24,9 @@ import org.apache.ambari.server.controller.utilities.StreamProvider; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; public class TestStreamProvider extends URLStreamProvider { @@ -43,12 +45,15 @@ public class TestStreamProvider extends URLStreamProvider { FILE_MAPPING.put("8745", "storm_rest_api_jmx.json"); } + private static String NN_HASTATE_ONLY_JMX = "hdfs_namenode_jmx_ha_only.json"; + /** * Delay to simulate response time. */ protected final long delay; private String lastSpec; + private List<String> specs = new ArrayList<>(); private boolean isLastSpecUpdated; @@ -64,11 +69,19 @@ public class TestStreamProvider extends URLStreamProvider { @Override public InputStream readFrom(String spec) throws IOException { + specs.add(spec); if (!isLastSpecUpdated) lastSpec = spec; isLastSpecUpdated = false; - String filename = FILE_MAPPING.get(getPort(spec)); + + String filename = null; + if (spec.endsWith(":50070/jmx?get=Hadoop:service=NameNode,name=FSNamesystem::tag.HAState")) { + filename = NN_HASTATE_ONLY_JMX; + } else { + filename = FILE_MAPPING.get(getPort(spec)); + } + if (filename == null) { throw new IOException("Can't find JMX source for " + spec); } @@ -87,6 +100,10 @@ public class TestStreamProvider extends URLStreamProvider { return lastSpec; } + public List<String> getSpecs() { + return specs; + } + private String getPort(String spec) { int colonIndex = spec.indexOf(":", 5); int slashIndex = spec.indexOf("/", colonIndex); http://git-wip-us.apache.org/repos/asf/ambari/blob/35ff7d79/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java index 4adea20..80d7438 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java @@ -24,9 +24,11 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -34,6 +36,7 @@ import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; +import org.apache.ambari.server.controller.internal.PropertyInfo; import org.apache.ambari.server.controller.internal.ResourceImpl; import org.apache.ambari.server.controller.jmx.JMXHostProvider; import org.apache.ambari.server.controller.jmx.JMXPropertyProvider; @@ -72,6 +75,15 @@ public class JMXPropertyProviderTest { protected static final String HOST_COMPONENT_STATE_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "state"); public static final int NUMBER_OF_RESOURCES = 400; + + public static final Map<String, Map<String, PropertyInfo>> jmxPropertyIds = PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent); + public static final Map<String, Map<String, PropertyInfo>> jmxPropertyIdsWithHAState; + + static { + jmxPropertyIdsWithHAState = new HashMap<>(jmxPropertyIds); + jmxPropertyIdsWithHAState.get("NAMENODE").put("metrics/dfs/FSNamesystem/HAState", new PropertyInfo("Hadoop:service=NameNode,name=FSNamesystem.tag#HAState", false, true)); + } + private static MetricPropertyProviderFactory metricPropertyProviderFactory; @BeforeClass @@ -126,6 +138,7 @@ public class JMXPropertyProviderTest { testPopulateResourcesUnhealthyResource(); testPopulateResourcesMany(); testPopulateResourcesTimeout(); + testPopulateResources_HAState_request(); } @Test @@ -139,6 +152,7 @@ public class JMXPropertyProviderTest { testPopulateResourcesUnhealthyResource(); testPopulateResourcesMany(); testPopulateResourcesTimeout(); + testPopulateResources_HAState_request(); } @Test @@ -152,6 +166,7 @@ public class JMXPropertyProviderTest { testPopulateResourcesUnhealthyResource(); testPopulateResourcesMany(); testPopulateResourcesTimeout(); + testPopulateResources_HAState_request(); } @Test(expected = AuthorizationException.class) @@ -167,6 +182,7 @@ public class JMXPropertyProviderTest { testPopulateResourcesUnhealthyResource(); testPopulateResourcesMany(); testPopulateResourcesTimeout(); + testPopulateResources_HAState_request(); } public void testPopulateResources() throws Exception { @@ -174,7 +190,7 @@ public class JMXPropertyProviderTest { TestJMXHostProvider hostProvider = new TestJMXHostProvider(false); TestMetricHostProvider metricsHostProvider = new TestMetricHostProvider(); JMXPropertyProvider propertyProvider = metricPropertyProviderFactory.createJMXPropertyProvider( - PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent), + jmxPropertyIdsWithHAState, streamProvider, hostProvider, metricsHostProvider, @@ -194,7 +210,10 @@ public class JMXPropertyProviderTest { Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet()); Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); - Assert.assertEquals(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx"), streamProvider.getLastSpec()); + List<String> expectedSpecs = new ArrayList<String>(); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx")); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx?get=Hadoop:service=NameNode,name=FSNamesystem::tag.HAState")); + Assert.assertEquals(expectedSpecs, streamProvider.getSpecs()); // see test/resources/hdfs_namenode_jmx.json for values Assert.assertEquals(13670605, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "ReceivedBytes"))); @@ -326,7 +345,10 @@ public class JMXPropertyProviderTest { Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); - Assert.assertEquals(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx"), streamProvider.getLastSpec()); + List<String> expectedSpecs = new ArrayList<String>(); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx")); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx?get=Hadoop:service=NameNode,name=FSNamesystem::tag.HAState")); + Assert.assertEquals(expectedSpecs, streamProvider.getSpecs()); // see test/resources/hdfs_namenode_jmx.json for values Assert.assertEquals(184320, resource.getPropertyValue("metrics/dfs/FSNamesystem/CapacityUsed")); @@ -334,6 +356,75 @@ public class JMXPropertyProviderTest { Assert.assertNull(resource.getPropertyValue("metrics/rpc/ReceivedBytes")); } + public void testPopulateResources_HAState_request() throws Exception { + TestStreamProvider streamProvider = new TestStreamProvider(); + TestJMXHostProvider hostProvider = new TestJMXHostProvider(false); + TestMetricHostProvider metricsHostProvider = new TestMetricHostProvider(); + + JMXPropertyProvider propertyProvider = metricPropertyProviderFactory.createJMXPropertyProvider( + jmxPropertyIdsWithHAState, + streamProvider, + hostProvider, + metricsHostProvider, + PropertyHelper.getPropertyId("HostRoles", "cluster_name"), + PropertyHelper.getPropertyId("HostRoles", "host_name"), + PropertyHelper.getPropertyId("HostRoles", "component_name"), + PropertyHelper.getPropertyId("HostRoles", "state")); + + // namenode + Resource resource = new ResourceImpl(Resource.Type.HostComponent); + resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); + resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-0e-34-e1.compute-1.internal"); + resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "NAMENODE"); + resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "STARTED"); + + // request with an empty set should get all supported properties + // only ask for one property + Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); + Request request = PropertyHelper.getReadRequest(Collections.singleton("metrics/dfs/FSNamesystem"), temporalInfoMap); + + Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); + + List<String> expectedSpecs = new ArrayList<String>(); + expectedSpecs.add(propertyProvider.getSpec("http","domu-12-31-39-0e-34-e1.compute-1.internal","50070","/jmx")); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx?get=Hadoop:service=NameNode,name=FSNamesystem::tag.HAState")); + + Assert.assertEquals(expectedSpecs, streamProvider.getSpecs()); + + // see test/resources/hdfs_namenode_jmx.json for values + Assert.assertEquals(184320, resource.getPropertyValue("metrics/dfs/FSNamesystem/CapacityUsed")); + Assert.assertEquals(21, resource.getPropertyValue("metrics/dfs/FSNamesystem/UnderReplicatedBlocks")); + Assert.assertEquals("customState", resource.getPropertyValue("metrics/dfs/FSNamesystem/HAState")); + Assert.assertNull(resource.getPropertyValue("metrics/rpc/ReceivedBytes")); + + // namenode + resource = new ResourceImpl(Resource.Type.HostComponent); + resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); + resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-0e-34-e1.compute-1.internal"); + resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "NAMENODE"); + resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "STARTED"); + + streamProvider.getSpecs().clear(); + + // request with an empty set should get all supported properties + // only ask for one property + temporalInfoMap = new HashMap<String, TemporalInfo>(); + request = PropertyHelper.getReadRequest(Collections.singleton("metrics/dfs/FSNamesystem/CapacityUsed"), temporalInfoMap); + + Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); + + // HAState isn't requested. It shouldn't be retrieved. + expectedSpecs.clear(); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx")); + Assert.assertEquals(expectedSpecs, streamProvider.getSpecs()); + + // see test/resources/hdfs_namenode_jmx.json for values + Assert.assertEquals(184320, resource.getPropertyValue("metrics/dfs/FSNamesystem/CapacityUsed")); + Assert.assertNull(resource.getPropertyValue("metrics/dfs/FSNamesystem/HAState")); + Assert.assertNull(resource.getPropertyValue("metrics/rpc/ReceivedBytes")); + + } + public void testPopulateResourcesWithUnknownPort() throws Exception { TestStreamProvider streamProvider = new TestStreamProvider(); TestJMXHostProvider hostProvider = new TestJMXHostProvider(true); @@ -360,7 +451,10 @@ public class JMXPropertyProviderTest { Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); - Assert.assertEquals(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx"), streamProvider.getLastSpec()); + List<String> expectedSpecs = new ArrayList<String>(); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx")); + expectedSpecs.add(propertyProvider.getSpec("http", "domu-12-31-39-0e-34-e1.compute-1.internal", "50070", "/jmx?get=Hadoop:service=NameNode,name=FSNamesystem::tag.HAState")); + Assert.assertEquals(expectedSpecs, streamProvider.getSpecs()); // see test/resources/hdfs_namenode_jmx.json for values Assert.assertEquals(13670605, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "ReceivedBytes"))); http://git-wip-us.apache.org/repos/asf/ambari/blob/35ff7d79/ambari-server/src/test/resources/hdfs_namenode_jmx_ha_only.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/hdfs_namenode_jmx_ha_only.json b/ambari-server/src/test/resources/hdfs_namenode_jmx_ha_only.json new file mode 100644 index 0000000..45b4f4b --- /dev/null +++ b/ambari-server/src/test/resources/hdfs_namenode_jmx_ha_only.json @@ -0,0 +1,7 @@ +{ + "beans" : [ { + "name" : "Hadoop:service=NameNode,name=FSNamesystem", + "modelerType" : "FSNamesystem", + "tag.HAState" : "customState" + } ] +} \ No newline at end of file
