Repository: ambari Updated Branches: refs/heads/branch-2.4 209eb5650 -> dae3e88ee refs/heads/trunk 042f26110 -> 82af4e0d8
AMBARI-16742. Storm REST metrics are not available when ui.port is changed (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/82af4e0d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/82af4e0d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/82af4e0d Branch: refs/heads/trunk Commit: 82af4e0d813e45c03d61de4bc111b2245a6d3d08 Parents: 042f261 Author: Andrew Onishuk <[email protected]> Authored: Thu May 19 14:18:40 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu May 19 14:18:40 2016 +0300 ---------------------------------------------------------------------- .../metrics/RestMetricsPropertyProvider.java | 16 +- .../common-services/STORM/0.10.0/metrics.json | 4 +- .../common-services/STORM/0.9.3/metrics.json | 4 +- .../RestMetricsPropertyProviderTest.java | 177 ++++++++++--------- 4 files changed, 105 insertions(+), 96 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/82af4e0d/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java index c579705..6f2a134 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.Type; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Hashtable; @@ -267,7 +268,7 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid * * @return determines REST port for service */ - private String resolvePort(Cluster cluster, String hostname, String componentName, + protected String resolvePort(Cluster cluster, String hostname, String componentName, Map<String, String> metricsProperties) throws AmbariException { String portConfigType = null; @@ -283,15 +284,18 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid Map<String, Map<String, String>> configTags = amc.findConfigurationTagsWithOverrides(cluster, hostname); if (configTags.containsKey(portConfigType)) { - Map<String, String> config = configTags.get(portConfigType); - if (config.containsKey(portPropertyName)) { + Map<String, Map<String, String>> properties = amc.getConfigHelper().getEffectiveConfigProperties(cluster, + Collections.singletonMap(portConfigType, configTags.get(portConfigType))); + Map<String, String> config = properties.get(portConfigType); + if (config != null && config.containsKey(portPropertyName)) { portStr = config.get(portPropertyName); } } } catch (AmbariException e) { - String message = String.format("Can not extract config tags for " + - "cluster = %s, hostname = %s", componentName, hostname); - LOG.warn(message); + String message = String.format("Can not extract configs for " + + "component = %s, hostname = %s, config type = %s, property name = %s", componentName, + hostname, portConfigType, portPropertyName); + LOG.warn(message, e); } if (portStr == null) { String message = String.format( http://git-wip-us.apache.org/repos/asf/ambari/blob/82af4e0d/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json b/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json index 3d0fdd3..a040f8a 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.10.0/metrics.json @@ -6,7 +6,7 @@ "properties": { "default_port": "8744", "port_config_type": "storm-site", - "port_property_name": "storm.port", + "port_property_name": "ui.port", "protocol": "http" }, "metrics": { @@ -62,7 +62,7 @@ "properties": { "default_port": "8744", "port_config_type": "storm-site", - "port_property_name": "storm.port", + "port_property_name": "ui.port", "protocol": "http" }, "metrics": { http://git-wip-us.apache.org/repos/asf/ambari/blob/82af4e0d/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json b/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json index 2b46faf..e739e3a 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.3/metrics.json @@ -6,7 +6,7 @@ "properties": { "default_port": "8744", "port_config_type": "storm-site", - "port_property_name": "storm.port", + "port_property_name": "ui.port", "protocol": "http" }, "metrics": { @@ -61,7 +61,7 @@ "properties": { "default_port": "8744", "port_config_type": "storm-site", - "port_property_name": "storm.port", + "port_property_name": "ui.port", "protocol": "http" }, "metrics": { http://git-wip-us.apache.org/repos/asf/ambari/blob/82af4e0d/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java index de3bc82..f78024f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java @@ -21,7 +21,6 @@ package org.apache.ambari.server.controller.metrics; import com.google.inject.Guice; import com.google.inject.Injector; -import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; @@ -36,15 +35,18 @@ import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.TemporalInfo; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.controller.utilities.StreamProvider; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.security.TestAuthenticationFactory; import org.apache.ambari.server.security.authorization.AuthorizationException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.stack.Metric; import org.apache.ambari.server.state.stack.MetricDefinition; +import org.easymock.EasyMock; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; @@ -58,8 +60,9 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; @@ -76,15 +79,17 @@ public class RestMetricsPropertyProviderTest { protected static final Map<String, String> metricsProperties = new HashMap<String, String>(); protected static final Map<String, Metric> componentMetrics = new HashMap<String, Metric>(); private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "cluster_name"); + private static final String DEFAULT_STORM_UI_PORT = "8745"; public static final int NUMBER_OF_RESOURCES = 400; private static Injector injector; private static Clusters clusters; private static Cluster c1; + private static AmbariManagementController amc; { - metricsProperties.put("default_port", "8745"); + metricsProperties.put("default_port", DEFAULT_STORM_UI_PORT); metricsProperties.put("port_config_type", "storm-site"); - metricsProperties.put("port_property_name", "storm.port"); + metricsProperties.put("port_property_name", "ui.port"); metricsProperties.put("protocol", "http"); componentMetrics.put("metrics/api/cluster/summary/tasks.total", new Metric("/api/cluster/summary##tasks.total", false, false, false, "unitless")); componentMetrics.put("metrics/api/cluster/summary/slots.total", new Metric("/api/cluster/summary##slots.total", false, false, false, "unitless")); @@ -109,23 +114,47 @@ public class RestMetricsPropertyProviderTest { // Setting up Mocks for Controller, Clusters etc, queried as part of user's Role context // while fetching Metrics. - AmbariManagementController amc = createNiceMock(AmbariManagementController.class); + amc = createNiceMock(AmbariManagementController.class); Field field = AmbariServer.class.getDeclaredField("clusterController"); field.setAccessible(true); field.set(null, amc); - Clusters clustersMock = createNiceMock(Clusters.class); - Cluster clusterMock = createNiceMock(Cluster.class); - expect(amc.getClusters()).andReturn(clustersMock).anyTimes(); - expect(clustersMock.getCluster(CLUSTER_NAME_PROPERTY_ID)).andReturn(clusterMock).anyTimes(); - expect(clusterMock.getResourceId()).andReturn(2L).anyTimes(); - try { - expect(clustersMock.getCluster(anyObject(String.class))).andReturn(clusterMock).anyTimes(); - } catch (AmbariException e) { - e.printStackTrace(); - } - replay(amc, clustersMock, clusterMock); + + ConfigHelper configHelperMock = createNiceMock(ConfigHelper.class); + expect(amc.getClusters()).andReturn(clusters).anyTimes(); + expect(amc.findConfigurationTagsWithOverrides(eq(c1), anyString())).andReturn(Collections.singletonMap("storm-site", + Collections.singletonMap("tag", "version1"))).anyTimes(); + expect(amc.getConfigHelper()).andReturn(configHelperMock).anyTimes(); + expect(configHelperMock.getEffectiveConfigProperties(eq(c1), + EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(Collections.singletonMap("storm-site", + Collections.singletonMap("ui.port", DEFAULT_STORM_UI_PORT))).anyTimes(); + replay(amc, configHelperMock); + } + + private RestMetricsPropertyProvider createRestMetricsPropertyProvider(MetricDefinition metricDefinition, + HashMap<String, Map<String, PropertyInfo>> componentMetrics, StreamProvider streamProvider, + TestMetricsHostProvider metricsHostProvider) throws Exception { + + RestMetricsPropertyProvider restMetricsPropertyProvider = new RestMetricsPropertyProvider( + injector, + metricDefinition.getProperties(), + componentMetrics, + streamProvider, + metricsHostProvider, + PropertyHelper.getPropertyId("HostRoles", "cluster_name"), + PropertyHelper.getPropertyId("HostRoles", "host_name"), + PropertyHelper.getPropertyId("HostRoles", "component_name"), + PropertyHelper.getPropertyId("HostRoles", "state"), + "STORM_REST_API" + ); + + Field field = RestMetricsPropertyProvider.class.getDeclaredField("amc"); + field.setAccessible(true); + field.set(restMetricsPropertyProvider, amc); + + return restMetricsPropertyProvider; } + @After public void clearAuthentication() { SecurityContextHolder.getContext().setAuthentication(null); @@ -181,7 +210,8 @@ public class RestMetricsPropertyProviderTest { testPopulateResourcesTimeout(); } - public void testPopulateResources() throws Exception { + @Test + public void testResolvePort() throws Exception { MetricDefinition metricDefinition = createNiceMock(MetricDefinition.class); expect(metricDefinition.getMetrics()).andReturn(componentMetrics); expect(metricDefinition.getType()).andReturn("org.apache.ambari.server.controller.metrics.RestMetricsPropertyProvider"); @@ -194,18 +224,41 @@ public class RestMetricsPropertyProviderTest { TestStreamProvider streamProvider = new TestStreamProvider(); TestMetricsHostProvider metricsHostProvider = new TestMetricsHostProvider(); - RestMetricsPropertyProvider restMetricsPropertyProvider = new RestMetricsPropertyProvider( - injector, - metricDefinition.getProperties(), - componentMetrics, - streamProvider, - metricsHostProvider, - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name"), - PropertyHelper.getPropertyId("HostRoles", "state"), - "STORM_REST_API"); + RestMetricsPropertyProvider restMetricsPropertyProvider = createRestMetricsPropertyProvider(metricDefinition, componentMetrics, streamProvider, + metricsHostProvider); + + // a property with a port doesn't exist, should return a default + Map<String, String> customMetricsProperties = new HashMap<>(metricsProperties); + customMetricsProperties.put("port_property_name", "wrong_property"); + String resolvedPort = restMetricsPropertyProvider.resolvePort(c1, "domu-12-31-39-0e-34-e1.compute-1.internal", + "STORM_REST_API", customMetricsProperties); + Assert.assertEquals(DEFAULT_STORM_UI_PORT, resolvedPort); + // a port property exists (8745). Should return it, not a default_port (8746) + customMetricsProperties = new HashMap<>(metricsProperties); + // custom default + customMetricsProperties.put("default_port", "8746"); + resolvedPort = restMetricsPropertyProvider.resolvePort(c1, "domu-12-31-39-0e-34-e1.compute-1.internal", + "STORM_REST_API", customMetricsProperties); + Assert.assertEquals(DEFAULT_STORM_UI_PORT, resolvedPort); + + } + + public void testPopulateResources() throws Exception { + MetricDefinition metricDefinition = createNiceMock(MetricDefinition.class); + expect(metricDefinition.getMetrics()).andReturn(componentMetrics); + expect(metricDefinition.getType()).andReturn("org.apache.ambari.server.controller.metrics.RestMetricsPropertyProvider"); + expect(metricDefinition.getProperties()).andReturn(metricsProperties); + replay(metricDefinition); + + Map<String, PropertyInfo> metrics = StackDefinedPropertyProvider.getPropertyInfo(metricDefinition); + HashMap<String, Map<String, PropertyInfo>> componentMetrics = new HashMap<String, Map<String, PropertyInfo>>(); + componentMetrics.put(WRAPPED_METRICS_KEY, metrics); + TestStreamProvider streamProvider = new TestStreamProvider(); + TestMetricsHostProvider metricsHostProvider = new TestMetricsHostProvider(); + + RestMetricsPropertyProvider restMetricsPropertyProvider = createRestMetricsPropertyProvider(metricDefinition, componentMetrics, streamProvider, + metricsHostProvider); Resource resource = new ResourceImpl(Resource.Type.HostComponent); @@ -242,18 +295,8 @@ public class RestMetricsPropertyProviderTest { TestStreamProvider streamProvider = new TestStreamProvider(); TestMetricsHostProvider metricsHostProvider = new TestMetricsHostProvider(); - RestMetricsPropertyProvider restMetricsPropertyProvider = new RestMetricsPropertyProvider( - injector, - metricDefinition.getProperties(), - componentMetrics, - streamProvider, - metricsHostProvider, - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name"), - PropertyHelper.getPropertyId("HostRoles", "state"), - "STORM_REST_API"); - + RestMetricsPropertyProvider restMetricsPropertyProvider = createRestMetricsPropertyProvider(metricDefinition, componentMetrics, streamProvider, + metricsHostProvider); Resource resource = new ResourceImpl(Resource.Type.HostComponent); @@ -283,18 +326,8 @@ public class RestMetricsPropertyProviderTest { TestStreamProvider streamProvider = new TestStreamProvider(); TestMetricsHostProvider metricsHostProvider = new TestMetricsHostProvider(); - RestMetricsPropertyProvider restMetricsPropertyProvider = new RestMetricsPropertyProvider( - injector, - metricDefinition.getProperties(), - componentMetrics, - streamProvider, - metricsHostProvider, - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name"), - PropertyHelper.getPropertyId("HostRoles", "state"), - "STORM_REST_API"); - + RestMetricsPropertyProvider restMetricsPropertyProvider = createRestMetricsPropertyProvider(metricDefinition, componentMetrics, streamProvider, + metricsHostProvider); Resource resource = new ResourceImpl(Resource.Type.HostComponent); @@ -328,18 +361,8 @@ public class RestMetricsPropertyProviderTest { TestStreamProvider streamProvider = new TestStreamProvider(); TestMetricsHostProvider metricsHostProvider = new TestMetricsHostProvider(); - RestMetricsPropertyProvider restMetricsPropertyProvider = new RestMetricsPropertyProvider( - injector, - metricDefinition.getProperties(), - componentMetrics, - streamProvider, - metricsHostProvider, - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name"), - PropertyHelper.getPropertyId("HostRoles", "state"), - "STORM_REST_API"); - + RestMetricsPropertyProvider restMetricsPropertyProvider = createRestMetricsPropertyProvider(metricDefinition, componentMetrics, streamProvider, + metricsHostProvider); Resource resource = new ResourceImpl(Resource.Type.HostComponent); @@ -371,17 +394,8 @@ public class RestMetricsPropertyProviderTest { Set<Resource> resources = new HashSet<Resource>(); - RestMetricsPropertyProvider restMetricsPropertyProvider = new RestMetricsPropertyProvider( - injector, - metricDefinition.getProperties(), - componentMetrics, - streamProvider, - metricsHostProvider, - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name"), - PropertyHelper.getPropertyId("HostRoles", "state"), - "STORM_REST_API"); + RestMetricsPropertyProvider restMetricsPropertyProvider = createRestMetricsPropertyProvider(metricDefinition, componentMetrics, streamProvider, + metricsHostProvider); for (int i = 0; i < NUMBER_OF_RESOURCES; ++i) { // strom_rest_api @@ -425,17 +439,8 @@ public class RestMetricsPropertyProviderTest { Set<Resource> resources = new HashSet<Resource>(); - RestMetricsPropertyProvider restMetricsPropertyProvider = new RestMetricsPropertyProvider( - injector, - metricDefinition.getProperties(), - componentMetrics, - streamProvider, - metricsHostProvider, - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name"), - PropertyHelper.getPropertyId("HostRoles", "state"), - "STORM_REST_API"); + RestMetricsPropertyProvider restMetricsPropertyProvider = createRestMetricsPropertyProvider(metricDefinition, componentMetrics, streamProvider, + metricsHostProvider); // set the provider timeout to 50 millis restMetricsPropertyProvider.setPopulateTimeout(50L);
