Repository: ambari Updated Branches: refs/heads/trunk bc17f9cf4 -> ddc656714
AMBARI-15745 - Create Alert For Reporting Potential Issues With Slow REST Responses (part2) (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ddc65671 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ddc65671 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ddc65671 Branch: refs/heads/trunk Commit: ddc656714914c34597eb4791920dd68f1430b242 Parents: bc17f9c Author: Jonathan Hurley <[email protected]> Authored: Fri Apr 8 12:32:59 2016 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Fri Apr 8 13:39:50 2016 -0400 ---------------------------------------------------------------------- .../alerts/AmbariPerformanceRunnable.java | 89 ++++++++++++-------- .../server/orm/dao/HostRoleCommandDAO.java | 9 +- ambari-server/src/main/resources/alerts.json | 18 ++++ .../alerts/AmbariPerformanceRunnableTest.java | 70 ++++++++++++++- 4 files changed, 147 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ddc65671/ambari-server/src/main/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnable.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnable.java b/ambari-server/src/main/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnable.java index 63fa12d..a69abeb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnable.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnable.java @@ -20,25 +20,25 @@ package org.apache.ambari.server.alerts; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; -import java.util.Set; +import java.util.Map; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.actionmanager.RequestStatus; +import org.apache.ambari.server.api.query.Query; +import org.apache.ambari.server.api.query.QueryImpl; +import org.apache.ambari.server.api.query.render.DefaultRenderer; +import org.apache.ambari.server.api.resources.ClusterResourceDefinition; import org.apache.ambari.server.api.services.BaseRequest; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.internal.AbstractControllerResourceProvider; -import org.apache.ambari.server.controller.internal.ClusterResourceProvider; -import org.apache.ambari.server.controller.spi.Predicate; -import org.apache.ambari.server.controller.spi.Request; -import org.apache.ambari.server.controller.spi.Resource.Type; -import org.apache.ambari.server.controller.spi.ResourceProvider; -import org.apache.ambari.server.controller.utilities.PredicateBuilder; -import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.controller.spi.ClusterController; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; +import org.apache.ambari.server.orm.dao.RequestDAO; import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; +import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken; import org.apache.ambari.server.state.Alert; import org.apache.ambari.server.state.AlertState; import org.apache.ambari.server.state.Cluster; @@ -51,9 +51,9 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.security.core.context.SecurityContextHolder; import com.google.inject.Inject; -import com.google.inject.Provider; /** * The {@link AmbariPerformanceRunnable} is used by the @@ -104,7 +104,7 @@ public class AmbariPerformanceRunnable extends AlertRunnable { * The {@link PerformanceArea} enumeration represents logical areas of * functionality to test for performance. */ - private enum PerformanceArea { + enum PerformanceArea { /** * Query for requests by {@link RequestStatus#IN_PROGRESS}. @@ -116,12 +116,32 @@ public class AmbariPerformanceRunnable extends AlertRunnable { */ @Override void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception { - runnable.m_actionManager.get().getRequestsByStatus(RequestStatus.IN_PROGRESS, + runnable.m_actionManager.getRequestsByStatus(RequestStatus.IN_PROGRESS, BaseRequest.DEFAULT_PAGE_SIZE, false); } }, /** + * Query for requests by {@link RequestStatus#IN_PROGRESS}. + */ + HRC_SUMMARY_STATUS("Database Access (Task Status Aggregation)", + "task.status.aggregation.warning.threshold", 3000, + "task.status.aggregation.critical.threshold", 5000) { + /** + * {@inheritDoc} + */ + @Override + void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception { + List<Long> requestIds = runnable.m_requestDAO.findAllRequestIds( + BaseRequest.DEFAULT_PAGE_SIZE, false); + + for (long requestId : requestIds) { + runnable.m_hostRoleCommandDAO.findAggregateCounts(requestId); + } + } + }, + + /** * Query through the REST API framework for a cluster. */ REST_API_GET_CLUSTER("REST API (Cluster)", @@ -132,25 +152,19 @@ public class AmbariPerformanceRunnable extends AlertRunnable { */ @Override void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception { - Type type = Type.Cluster; - ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( - type, PropertyHelper.getPropertyIds(type), PropertyHelper.getKeyPropertyIds(type), - runnable.m_amc.get()); - - Set<String> propertyIds = new HashSet<String>(); - - propertyIds.add(ClusterResourceProvider.CLUSTER_ID_PROPERTY_ID); - propertyIds.add(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID); + // Set authenticated user so that authorization checks will pass + InternalAuthenticationToken authenticationToken = new InternalAuthenticationToken("admin"); + authenticationToken.setAuthenticated(true); + SecurityContextHolder.getContext().setAuthentication(authenticationToken); // create the request - Request request = PropertyHelper.getReadRequest(propertyIds); + Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>(); + mapIds.put(Resource.Type.Cluster, cluster.getClusterName()); - // build the predicate for this cluster - Predicate predicate = new PredicateBuilder().property( - ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID).equals( - cluster.getClusterName()).toPredicate(); - - provider.getResources(request, predicate); + ClusterController clusterController = ClusterControllerHelper.getClusterController(); + Query query = new QueryImpl(mapIds, new ClusterResourceDefinition(), clusterController); + query.setRenderer(new DefaultRenderer()); + query.execute(); } }; @@ -222,13 +236,22 @@ public class AmbariPerformanceRunnable extends AlertRunnable { } /** - * Used for querying for requests by status. + * Used for getting the most recent requests. + */ + @Inject + private RequestDAO m_requestDAO; + + /** + * Used for executing queries which are known to potentially take a long time. */ @Inject - private Provider<ActionManager> m_actionManager; + private HostRoleCommandDAO m_hostRoleCommandDAO; + /** + * Used for querying for requests by status. + */ @Inject - private Provider<AmbariManagementController> m_amc; + private ActionManager m_actionManager; /** * Constructor. http://git-wip-us.apache.org/repos/asf/ambari/blob/ddc65671/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java index 2e35001..1768f21 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java @@ -23,7 +23,6 @@ import static org.apache.ambari.server.orm.dao.DaoUtils.ORACLE_LIST_LIMIT; import java.text.MessageFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -318,7 +317,7 @@ public class HostRoleCommandDAO { "HostRoleCommandEntity.findByRequestIdAndStatuses", HostRoleCommandEntity.class); query.setParameter("requestId", requestId); query.setParameter("statuses", statuses); - List results = query.getResultList(); + List<HostRoleCommandEntity> results = query.getResultList(); return results; } @@ -675,6 +674,7 @@ public class HostRoleCommandDAO { * the request id * @return the map of stage-to-summary objects */ + @RequiresSession public Map<Long, HostRoleCommandStatusSummaryDTO> findAggregateCounts(Long requestId) { if (!hostRoleCommandStatusSummaryCacheEnabled) { return loadAggregateCounts(requestId); @@ -706,16 +706,17 @@ public class HostRoleCommandDAO { * @param requestId upgrade request id * @return Most recent task failure during stack upgrade, or null if one doesn't exist. */ + @RequiresSession public HostRoleCommandEntity findMostRecentFailure(Long requestId) { TypedQuery<HostRoleCommandEntity> query = entityManagerProvider.get().createNamedQuery( "HostRoleCommandEntity.findTasksByStatusesOrderByIdDesc", HostRoleCommandEntity.class); query.setParameter("requestId", requestId); query.setParameter("statuses", HostRoleStatus.STACK_UPGRADE_FAILED_STATUSES); - List results = query.getResultList(); + List<HostRoleCommandEntity> results = query.getResultList(); if (!results.isEmpty()) { - HostRoleCommandEntity candidate = (HostRoleCommandEntity) results.get(0); + HostRoleCommandEntity candidate = results.get(0); // Ensure that there are no other completed tasks in a future stage to avoid returning an old error. // During Express Upgrade, we can run multiple commands in the same stage, so it's possible to have http://git-wip-us.apache.org/repos/asf/ambari/blob/ddc65671/ambari-server/src/main/resources/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/alerts.json b/ambari-server/src/main/resources/alerts.json index 27ae76d..3b6ebbb 100644 --- a/ambari-server/src/main/resources/alerts.json +++ b/ambari-server/src/main/resources/alerts.json @@ -57,6 +57,24 @@ "threshold": "CRITICAL" }, { + "name": "task.status.aggregation.warning.threshold", + "display_name": "Warning", + "value": 3000, + "type": "NUMERIC", + "description": "The time to calculate a request's status from its tasks before a warning alert is triggered.", + "units": "ms", + "threshold": "WARNING" + }, + { + "name": "task.status.aggregation.critical.threshold", + "display_name": "Critical", + "value": 5000, + "type": "NUMERIC", + "description": "The time to calculate a request's status from its tasks before a critical alert is triggered.", + "units": "ms", + "threshold": "CRITICAL" + }, + { "name": "rest.api.cluster.warning.threshold", "display_name": "Warning", "value": 5000, http://git-wip-us.apache.org/repos/asf/ambari/blob/ddc65671/ambari-server/src/test/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnableTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnableTest.java b/ambari-server/src/test/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnableTest.java index 31ed745..7b1a5a2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnableTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnableTest.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.alerts; import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; @@ -33,8 +34,11 @@ import java.util.Map; import javax.persistence.EntityManager; import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.alerts.AmbariPerformanceRunnable.PerformanceArea; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.internal.ClusterResourceProvider; +import org.apache.ambari.server.controller.spi.ClusterController; +import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.events.AlertEvent; import org.apache.ambari.server.events.AlertReceivedEvent; import org.apache.ambari.server.events.MockEventListener; @@ -43,6 +47,7 @@ import org.apache.ambari.server.orm.DBAccessor; import org.apache.ambari.server.orm.dao.AlertDefinitionDAO; import org.apache.ambari.server.orm.dao.AlertsDAO; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; +import org.apache.ambari.server.orm.dao.RequestDAO; import org.apache.ambari.server.orm.entities.AlertCurrentEntity; import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; import org.apache.ambari.server.state.Alert; @@ -57,6 +62,10 @@ import org.easymock.EasyMock; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import com.google.common.eventbus.EventBus; import com.google.inject.Binder; @@ -67,6 +76,9 @@ import com.google.inject.Module; /** * Tests {@link AmbariPerformanceRunnable}. */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({ ClusterControllerHelper.class, AmbariPerformanceRunnable.class, + PerformanceArea.class }) public class AmbariPerformanceRunnableTest { private final static long CLUSTER_ID = 1; @@ -148,7 +160,13 @@ public class AmbariPerformanceRunnableTest { AlertDefinitionFactory factory = m_injector.getInstance(AlertDefinitionFactory.class); expect(factory.coerce(EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(definition).atLeastOnce(); - replay(m_definition, m_cluster, m_clusters, m_definitionDao, m_alertsDao, factory); + ClusterController clusterController = m_injector.getInstance(ClusterController.class); + PowerMock.mockStatic(ClusterControllerHelper.class); + expect(ClusterControllerHelper.getClusterController()).andReturn(clusterController); + PowerMock.replay(ClusterControllerHelper.class); + + replay(m_definition, m_cluster, m_clusters, m_definitionDao, m_alertsDao, factory, + clusterController); } /** @@ -159,10 +177,52 @@ public class AmbariPerformanceRunnableTest { } /** + * Tests that the event is triggerd with a status of OK if all performance + * areas pass. + */ + @Test + public void testAlertFiresOKEvent() { + // mock the entire enum so that no problems are reported + PowerMock.mockStatic(PerformanceArea.class); + expect(PerformanceArea.values()).andReturn(new PerformanceArea[0]); + PowerMock.replay(PerformanceArea.class); + + // instantiate and inject mocks + AmbariPerformanceRunnable runnable = new AmbariPerformanceRunnable( + m_definition.getDefinitionName()); + + m_injector.injectMembers(runnable); + + // run the alert + runnable.run(); + + assertEquals(1, m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class)); + + List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class); + assertEquals(1, events.size()); + + AlertReceivedEvent event = (AlertReceivedEvent) events.get(0); + Alert alert = event.getAlert(); + assertEquals("AMBARI", alert.getService()); + assertEquals("AMBARI_SERVER", alert.getComponent()); + assertEquals(AlertState.OK, alert.getState()); + assertEquals(DEFINITION_NAME, alert.getName()); + + verify(m_cluster, m_clusters, m_definitionDao); + } + + /** * Tests that the event is triggerd with a status of UNKNOWN. */ @Test - public void testAlertFiresEvents() { + public void testAlertFiresUnknownEvent() { + // mock one area, leaving others to fail + RequestDAO requestDAO = m_injector.getInstance(RequestDAO.class); + expect(requestDAO.findAllRequestIds(EasyMock.anyInt(), EasyMock.anyBoolean())).andReturn( + new ArrayList<Long>()); + + replay(requestDAO); + // instantiate and inject mocks AmbariPerformanceRunnable runnable = new AmbariPerformanceRunnable( m_definition.getDefinitionName()); @@ -185,6 +245,10 @@ public class AmbariPerformanceRunnableTest { assertEquals(AlertState.UNKNOWN, alert.getState()); assertEquals(DEFINITION_NAME, alert.getName()); + // verify that even though there is 1 UNKNOWN, there should also be 1 OK as + // well + assertTrue(alert.getText().contains("(OK)")); + verify(m_cluster, m_clusters, m_definitionDao); } @@ -227,6 +291,8 @@ public class AmbariPerformanceRunnableTest { binder.bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class)); binder.bind(AlertDefinitionFactory.class).toInstance(createNiceMock(AlertDefinitionFactory.class)); binder.bind(ClusterResourceProvider.class).toInstance(createNiceMock(ClusterResourceProvider.class)); + binder.bind(ClusterController.class).toInstance(createNiceMock(ClusterController.class)); + binder.bind(RequestDAO.class).toInstance(createNiceMock(RequestDAO.class)); } } }
