Updated Branches: refs/heads/trunk 52164b180 -> d43f37e52
Powering off RM node increases API latency by a factor of 6. (odiachenko) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/d43f37e5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/d43f37e5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/d43f37e5 Branch: refs/heads/trunk Commit: d43f37e52102d561b2958da9ea5e37946bc92a8b Parents: 52164b1 Author: Oleksandr Diachenko <[email protected]> Authored: Wed Sep 11 15:47:24 2013 +0300 Committer: Oleksandr Diachenko <[email protected]> Committed: Wed Sep 11 15:47:24 2013 +0300 ---------------------------------------------------------------------- .../controller/ganglia/GangliaHostProvider.java | 22 ++++++ .../ganglia/GangliaPropertyProvider.java | 15 +++++ .../internal/AbstractProviderModule.java | 66 ++++++++++++++++++ .../controller/internal/URLStreamProvider.java | 5 -- .../ganglia/GangliaPropertyProviderTest.java | 71 ++++++++++++++++++++ .../GangliaReportPropertyProviderTest.java | 13 ++++ 6 files changed, 187 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/d43f37e5/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaHostProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaHostProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaHostProvider.java index 37e1e05..51721a4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaHostProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaHostProvider.java @@ -34,4 +34,26 @@ public interface GangliaHostProvider { * @throws SystemException if unable to get the Ganglia server host name */ public String getGangliaCollectorHostName(String clusterName) throws SystemException; + + /** + * Get the status of Ganglia server host for the given cluster name. + * + * @param clusterName the cluster name + * + * @return true if heartbeat with Ganglia server host wasn't lost + * + * @throws SystemException if unable to get the status of Ganglia server host + */ + public boolean isGangliaCollectorHostLive(String clusterName) throws SystemException; + + /** + * Get the status of Ganglia server component for the given cluster name. + * + * @param clusterName the cluster name + * + * @return true if Ganglia server component is started + * + * @throws SystemException if unable to get the status of Ganglia server component + */ + public boolean isGangliaCollectorComponentLive(String clusterName) throws SystemException; } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/d43f37e5/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java index 6cc981c..62ec61c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java @@ -428,6 +428,20 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider { requestAll ? Collections.<String>emptySet() : metrics.keySet(), temporalInfo); BufferedReader reader = null; try { + + //Check if host is live + if (!hostProvider.isGangliaCollectorHostLive(clusterName)) { + LOG.info("Ganglia host is not live"); + return Collections.emptySet(); + } + + //Check if Ganglia server component is live + if (!hostProvider.isGangliaCollectorComponentLive(clusterName)) { + LOG.info("Ganglia server component is not live"); + return Collections.emptySet(); + } + + reader = new BufferedReader(new InputStreamReader( getStreamProvider().readFrom(spec))); @@ -515,6 +529,7 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider { return Collections.emptySet(); } + /** * Populate the given resource with the given Ganglia metric. * http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/d43f37e5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java index 90261fc..0caf9bc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java @@ -18,8 +18,14 @@ package org.apache.ambari.server.controller.internal; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.Role; import org.apache.ambari.server.configuration.ComponentSSLConfiguration; import org.apache.ambari.server.controller.AmbariServer; +import org.apache.ambari.server.controller.HostRequest; +import org.apache.ambari.server.controller.HostResponse; +import org.apache.ambari.server.controller.ServiceComponentHostRequest; +import org.apache.ambari.server.controller.ServiceComponentHostResponse; import org.apache.ambari.server.controller.ganglia.GangliaComponentPropertyProvider; import org.apache.ambari.server.controller.ganglia.GangliaHostComponentPropertyProvider; import org.apache.ambari.server.controller.ganglia.GangliaHostPropertyProvider; @@ -34,7 +40,10 @@ import org.apache.ambari.server.controller.AmbariManagementController; import com.google.inject.Inject; import org.apache.ambari.server.controller.utilities.StreamProvider; import org.apache.ambari.server.state.DesiredConfig; +import org.apache.ambari.server.state.HostState; import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.State; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -231,6 +240,63 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource checkInit(); return clusterGangliaCollectorMap.get(clusterName); } + + @Override + public boolean isGangliaCollectorHostLive(String clusterName) throws SystemException { + + HostResponse gangliaCollectorHost = null; + + try { + HostRequest hostRequest = new HostRequest(null, clusterName, Collections.<String, String>emptyMap()); + Set<HostResponse> hosts = managementController.getHosts(Collections.singleton(hostRequest)); + + final String gangliaCollectorHostName = getGangliaCollectorHostName(clusterName); + + gangliaCollectorHost = (HostResponse) CollectionUtils.find(hosts, new org.apache.commons.collections.Predicate() { + + @Override + public boolean evaluate(Object hostResponse) { + return ((HostResponse) hostResponse).getHostname().equals(gangliaCollectorHostName); + } + }); + } catch (AmbariException e) { + LOG.debug("Error checking of Ganglia server host live status: ", e); + return false; + } + + LOG.debug("Host state: " + gangliaCollectorHost.getHostState()); + + return !gangliaCollectorHost.getHostState().equals(HostState.HEARTBEAT_LOST.name()); + } + + @Override + public boolean isGangliaCollectorComponentLive(String clusterName) throws SystemException { + + + ServiceComponentHostResponse gangliaCollectorHostComponent = null; + + try { + final String gangliaCollectorHostName = getGangliaCollectorHostName(clusterName); + ServiceComponentHostRequest componentRequest = new ServiceComponentHostRequest(clusterName, "GANGLIA", + Role.GANGLIA_SERVER.name(), + gangliaCollectorHostName, + Collections.<String, String>emptyMap(), null); + + Set<ServiceComponentHostResponse> hostComponents = managementController.getHostComponents(Collections.singleton(componentRequest)); + gangliaCollectorHostComponent = (ServiceComponentHostResponse) CollectionUtils.find(hostComponents, + new org.apache.commons.collections.Predicate() { + @Override + public boolean evaluate(Object arg0) { + return ((ServiceComponentHostResponse) arg0).getHostname().equals(gangliaCollectorHostName); + } + }); + } catch (AmbariException e) { + LOG.debug("Error checking of Ganglia server host component state: ", e); + return false; + } + + return gangliaCollectorHostComponent.getLiveState().equals(State.STARTED.name()); + } // ----- utility methods --------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/d43f37e5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java index db52a4f..1bce033 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java @@ -35,12 +35,7 @@ import javax.net.ssl.TrustManagerFactory; import org.apache.ambari.server.controller.utilities.StreamProvider; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.message.BasicHeader; /** * URL based implementation of a stream provider. http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/d43f37e5/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java index aa05a5b..42130c1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java @@ -31,6 +31,11 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.anyObject; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; + import java.util.Arrays; import java.util.Collection; @@ -45,6 +50,7 @@ import java.util.Set; * Test the Ganglia property provider. */ @RunWith(Parameterized.class) +@PrepareForTest({ GangliaHostProvider.class }) public class GangliaPropertyProviderTest { private static final String PROPERTY_ID = PropertyHelper.getPropertyId("metrics/jvm", "gcCount"); @@ -161,6 +167,47 @@ public class GangliaPropertyProviderTest { Assert.assertNotNull(resource.getPropertyValue(shuffle_output_bytes)); Assert.assertNotNull(resource.getPropertyValue(shuffle_success_outputs)); } + + @Test + public void testPopulateResources_checkHostComponent() throws Exception { + TestStreamProvider streamProvider = new TestStreamProvider("temporal_ganglia_data.txt"); + GangliaHostProvider hostProvider = PowerMock.createPartialMock(GangliaHostProvider.class, + "isGangliaCollectorHostLive", "isGangliaCollectorComponentLive"); + + GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider( + PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, PropertyHelper.MetricsVersion.HDP1), + streamProvider, + configuration, + hostProvider, + CLUSTER_NAME_PROPERTY_ID, + HOST_NAME_PROPERTY_ID, + COMPONENT_NAME_PROPERTY_ID); + + // datanode + Resource resource = new ResourceImpl(Resource.Type.HostComponent); + + resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal"); + resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "DATANODE"); + + // only ask for one property + Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); + temporalInfoMap.put(PROPERTY_ID, new TemporalInfoImpl(10L, 20L, 1L)); + Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID), temporalInfoMap); + + expect(hostProvider.getGangliaCollectorHostName(anyObject(String.class))).andReturn("ganglia-host"); + expect(hostProvider.isGangliaCollectorComponentLive(anyObject(String.class))).andReturn(true).once(); + expect(hostProvider.isGangliaCollectorHostLive(anyObject(String.class))).andReturn(true).once(); + + + PowerMock.replay(hostProvider); + + Set<Resource> populateResources = propertyProvider.populateResources(Collections.singleton(resource), request, null); + + PowerMock.verify(hostProvider); + + Assert.assertEquals(1, populateResources.size()); + + } @Test @@ -719,9 +766,33 @@ public class GangliaPropertyProviderTest { private static class TestGangliaHostProvider implements GangliaHostProvider { + private boolean isHostLive; + private boolean isComponentLive; + + public TestGangliaHostProvider() { + this(true, true); + } + + public TestGangliaHostProvider(boolean isHostLive, boolean isComponentLive) { + this.isHostLive = isHostLive; + this.isComponentLive = isComponentLive; + } + @Override public String getGangliaCollectorHostName(String clusterName) { return "domU-12-31-39-0E-34-E1.compute-1.internal"; } + + @Override + public boolean isGangliaCollectorHostLive(String clusterName) + throws SystemException { + return isHostLive; + } + + @Override + public boolean isGangliaCollectorComponentLive(String clusterName) + throws SystemException { + return isComponentLive; + } } } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/d43f37e5/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaReportPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaReportPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaReportPropertyProviderTest.java index 243e630..5b13e31 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaReportPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaReportPropertyProviderTest.java @@ -23,6 +23,7 @@ import org.apache.ambari.server.controller.internal.ResourceImpl; import org.apache.ambari.server.controller.internal.TemporalInfoImpl; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.TemporalInfo; import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.junit.Assert; @@ -106,5 +107,17 @@ public class GangliaReportPropertyProviderTest { public String getGangliaCollectorHostName(String clusterName) { return "domU-12-31-39-0E-34-E1.compute-1.internal"; } + + @Override + public boolean isGangliaCollectorHostLive(String clusterName) + throws SystemException { + return true; + } + + @Override + public boolean isGangliaCollectorComponentLive(String clusterName) + throws SystemException { + return true; + } } }
