http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 571c96f..db867a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -1081,6 +1082,8 @@ public class TestClientRMService { .thenThrow(new IOException("queue does not exist")); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); + SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); + when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext, yarnScheduler); when(rmContext.getRMApps()).thenReturn(apps);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 15e45c4..7d92827 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -117,6 +117,10 @@ public abstract class MockAsm extends MockApps { throw new UnsupportedOperationException("Not supported yet."); } @Override + public String getOriginalTrackingUrl() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public int getMaxAppAttempts() { throw new UnsupportedOperationException("Not supported yet."); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java new file mode 100644 index 0000000..ec20af3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.metrics; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.EnumSet; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestSystemMetricsPublisher { + + private static ApplicationHistoryServer timelineServer; + private static SystemMetricsPublisher metricsPublisher; + private static TimelineStore store; + + @BeforeClass + public static void setup() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true); + conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, + MemoryTimelineStore.class, TimelineStore.class); + conf.setInt( + YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, + 2); + + timelineServer = new ApplicationHistoryServer(); + timelineServer.init(conf); + timelineServer.start(); + store = timelineServer.getTimelineStore(); + + metricsPublisher = new SystemMetricsPublisher(); + metricsPublisher.init(conf); + metricsPublisher.start(); + } + + @AfterClass + public static void tearDown() throws Exception { + if (metricsPublisher != null) { + metricsPublisher.stop(); + } + if (timelineServer != null) { + timelineServer.stop(); + } + AHSWebApp.resetInstance(); + } + + @Test(timeout = 10000) + public void testPublishApplicationMetrics() throws Exception { + ApplicationId appId = ApplicationId.newInstance(0, 1); + RMApp app = createRMApp(appId); + metricsPublisher.appCreated(app, app.getStartTime()); + metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime()); + TimelineEntity entity = null; + do { + entity = + store.getEntity(appId.toString(), + ApplicationMetricsConstants.ENTITY_TYPE, + EnumSet.allOf(Field.class)); + // ensure two events are both published before leaving the loop + } while (entity == null || entity.getEvents().size() < 2); + // verify all the fields + Assert.assertEquals(ApplicationMetricsConstants.ENTITY_TYPE, + entity.getEntityType()); + Assert + .assertEquals(app.getApplicationId().toString(), entity.getEntityId()); + Assert + .assertEquals( + app.getName(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.NAME_ENTITY_INFO)); + Assert.assertEquals(app.getQueue(), + entity.getOtherInfo() + .get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)); + Assert + .assertEquals( + app.getUser(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.USER_ENTITY_INFO)); + Assert + .assertEquals( + app.getApplicationType(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.TYPE_ENTITY_INFO)); + Assert.assertEquals(app.getSubmitTime(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO)); + boolean hasCreatedEvent = false; + boolean hasFinishedEvent = false; + for (TimelineEvent event : entity.getEvents()) { + if (event.getEventType().equals( + ApplicationMetricsConstants.CREATED_EVENT_TYPE)) { + hasCreatedEvent = true; + Assert.assertEquals(app.getStartTime(), event.getTimestamp()); + } else if (event.getEventType().equals( + ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) { + hasFinishedEvent = true; + Assert.assertEquals(app.getFinishTime(), event.getTimestamp()); + Assert.assertEquals( + app.getDiagnostics().toString(), + event.getEventInfo().get( + ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)); + Assert.assertEquals( + app.getFinalApplicationStatus().toString(), + event.getEventInfo().get( + ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)); + Assert.assertEquals(YarnApplicationState.FINISHED.toString(), event + .getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO)); + } + } + Assert.assertTrue(hasCreatedEvent && hasFinishedEvent); + } + + @Test(timeout = 10000) + public void testPublishAppAttemptMetrics() throws Exception { + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); + RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId); + metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L); + metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, + Integer.MAX_VALUE + 2L); + TimelineEntity entity = null; + do { + entity = + store.getEntity(appAttemptId.toString(), + AppAttemptMetricsConstants.ENTITY_TYPE, + EnumSet.allOf(Field.class)); + // ensure two events are both published before leaving the loop + } while (entity == null || entity.getEvents().size() < 2); + // verify all the fields + Assert.assertEquals(AppAttemptMetricsConstants.ENTITY_TYPE, + entity.getEntityType()); + Assert.assertEquals(appAttemptId.toString(), entity.getEntityId()); + Assert.assertEquals( + appAttemptId.getApplicationId().toString(), + entity.getPrimaryFilters() + .get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator() + .next()); + boolean hasRegisteredEvent = false; + boolean hasFinishedEvent = false; + for (TimelineEvent event : entity.getEvents()) { + if (event.getEventType().equals( + AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) { + hasRegisteredEvent = true; + Assert.assertEquals(appAttempt.getHost(), + event.getEventInfo() + .get(AppAttemptMetricsConstants.HOST_EVENT_INFO)); + Assert + .assertEquals(appAttempt.getRpcPort(), + event.getEventInfo().get( + AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO)); + Assert.assertEquals( + appAttempt.getMasterContainer().getId().toString(), + event.getEventInfo().get( + AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)); + } else if (event.getEventType().equals( + AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) { + hasFinishedEvent = true; + Assert.assertEquals(appAttempt.getDiagnostics(), event.getEventInfo() + .get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)); + Assert.assertEquals(appAttempt.getTrackingUrl(), event.getEventInfo() + .get(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO)); + Assert.assertEquals( + appAttempt.getOriginalTrackingUrl(), + event.getEventInfo().get( + AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO)); + Assert.assertEquals( + appAttempt.getFinalApplicationStatus().toString(), + event.getEventInfo().get( + AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO)); + Assert.assertEquals( + YarnApplicationAttemptState.FINISHED.toString(), + event.getEventInfo().get( + AppAttemptMetricsConstants.STATE_EVENT_INFO)); + } + } + Assert.assertTrue(hasRegisteredEvent && hasFinishedEvent); + } + + @Test(timeout = 10000) + public void testPublishContainerMetrics() throws Exception { + ContainerId containerId = + ContainerId.newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(0, 1), 1), 1); + RMContainer container = createRMContainer(containerId); + metricsPublisher.containerCreated(container, container.getCreationTime()); + metricsPublisher.containerFinished(container, container.getFinishTime()); + TimelineEntity entity = null; + do { + entity = + store.getEntity(containerId.toString(), + ContainerMetricsConstants.ENTITY_TYPE, + EnumSet.allOf(Field.class)); + // ensure two events are both published before leaving the loop + } while (entity == null || entity.getEvents().size() < 2); + // verify all the fields + Assert.assertEquals(ContainerMetricsConstants.ENTITY_TYPE, + entity.getEntityType()); + Assert.assertEquals(containerId.toString(), entity.getEntityId()); + Assert.assertEquals( + containerId.getApplicationAttemptId().toString(), + entity.getPrimaryFilters() + .get(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER).iterator() + .next()); + Assert.assertEquals( + container.getAllocatedNode().getHost(), + entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO)); + Assert.assertEquals( + container.getAllocatedNode().getPort(), + entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO)); + Assert.assertEquals( + container.getAllocatedResource().getMemory(), + entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO)); + Assert.assertEquals( + container.getAllocatedResource().getVirtualCores(), + entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO)); + Assert.assertEquals( + container.getAllocatedPriority().getPriority(), + entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO)); + boolean hasCreatedEvent = false; + boolean hasFinishedEvent = false; + for (TimelineEvent event : entity.getEvents()) { + if (event.getEventType().equals( + ContainerMetricsConstants.CREATED_EVENT_TYPE)) { + hasCreatedEvent = true; + Assert.assertEquals(container.getCreationTime(), event.getTimestamp()); + } else if (event.getEventType().equals( + ContainerMetricsConstants.FINISHED_EVENT_TYPE)) { + hasFinishedEvent = true; + Assert.assertEquals(container.getFinishTime(), event.getTimestamp()); + Assert.assertEquals( + container.getDiagnosticsInfo(), + event.getEventInfo().get( + ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)); + Assert.assertEquals( + container.getContainerExitStatus(), + event.getEventInfo().get( + ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO)); + Assert.assertEquals(container.getContainerState().toString(), event + .getEventInfo().get(ContainerMetricsConstants.STATE_EVENT_INFO)); + } + } + Assert.assertTrue(hasCreatedEvent && hasFinishedEvent); + } + + private static RMApp createRMApp(ApplicationId appId) { + RMApp app = mock(RMApp.class); + when(app.getApplicationId()).thenReturn(appId); + when(app.getName()).thenReturn("test app"); + when(app.getApplicationType()).thenReturn("test app type"); + when(app.getUser()).thenReturn("test user"); + when(app.getQueue()).thenReturn("test queue"); + when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L); + when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L); + when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L); + when(app.getDiagnostics()).thenReturn( + new StringBuilder("test diagnostics info")); + RMAppAttempt appAttempt = mock(RMAppAttempt.class); + when(appAttempt.getAppAttemptId()).thenReturn( + ApplicationAttemptId.newInstance(appId, 1)); + when(app.getCurrentAppAttempt()).thenReturn(appAttempt); + when(app.getFinalApplicationStatus()).thenReturn( + FinalApplicationStatus.UNDEFINED); + return app; + } + + private static RMAppAttempt createRMAppAttempt( + ApplicationAttemptId appAttemptId) { + RMAppAttempt appAttempt = mock(RMAppAttempt.class); + when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId); + when(appAttempt.getHost()).thenReturn("test host"); + when(appAttempt.getRpcPort()).thenReturn(-100); + Container container = mock(Container.class); + when(container.getId()) + .thenReturn(ContainerId.newInstance(appAttemptId, 1)); + when(appAttempt.getMasterContainer()).thenReturn(container); + when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info"); + when(appAttempt.getTrackingUrl()).thenReturn("test tracking url"); + when(appAttempt.getOriginalTrackingUrl()).thenReturn( + "test original tracking url"); + when(appAttempt.getFinalApplicationStatus()).thenReturn( + FinalApplicationStatus.UNDEFINED); + return appAttempt; + } + + private static RMContainer createRMContainer(ContainerId containerId) { + RMContainer container = mock(RMContainer.class); + when(container.getContainerId()).thenReturn(containerId); + when(container.getAllocatedNode()).thenReturn( + NodeId.newInstance("test host", -100)); + when(container.getAllocatedResource()).thenReturn( + Resource.newInstance(-1, -1)); + when(container.getAllocatedPriority()).thenReturn(Priority.UNDEFINED); + when(container.getCreationTime()).thenReturn(Integer.MAX_VALUE + 1L); + when(container.getFinishTime()).thenReturn(Integer.MAX_VALUE + 2L); + when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info"); + when(container.getContainerExitStatus()).thenReturn(-1); + when(container.getContainerState()).thenReturn(ContainerState.COMPLETE); + return container; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index b63d2fe..2fff718 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -50,6 +50,7 @@ public class MockRMApp implements RMApp { int failCount = 0; ApplicationId id; String url = null; + String oUrl = null; StringBuilder diagnostics = new StringBuilder(); RMAppAttempt attempt; int maxAppAttempts = 1; @@ -184,6 +185,15 @@ public class MockRMApp implements RMApp { } @Override + public String getOriginalTrackingUrl() { + return oUrl; + } + + public void setOriginalTrackingUrl(String oUrl) { + this.oUrl = oUrl; + } + + @Override public StringBuilder getDiagnostics() { return diagnostics; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 5874b5d..0874968 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; @@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; @@ -94,6 +96,7 @@ public class TestRMAppTransitions { private DrainDispatcher rmDispatcher; private RMStateStore store; private RMApplicationHistoryWriter writer; + private SystemMetricsPublisher publisher; private YarnScheduler scheduler; private TestSchedulerEventDispatcher schedulerDispatcher; @@ -203,6 +206,8 @@ public class TestRMAppTransitions { new ClientToAMTokenSecretManagerInRM(), writer); ((RMContextImpl)realRMContext).setStateStore(store); + publisher = mock(SystemMetricsPublisher.class); + ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher); this.rmContext = spy(realRMContext); @@ -354,6 +359,7 @@ public class TestRMAppTransitions { ApplicationSubmissionContext submissionContext) throws IOException { RMApp application = createNewTestApp(submissionContext); verify(writer).applicationStarted(any(RMApp.class)); + verify(publisher).appCreated(any(RMApp.class), anyLong()); // NEW => NEW_SAVING event RMAppEventType.START RMAppEvent event = new RMAppEvent(application.getApplicationId(), RMAppEventType.START); @@ -477,6 +483,7 @@ public class TestRMAppTransitions { // reset the counter of Mockito.verify reset(writer); + reset(publisher); // test app fails after 1 app attempt failure LOG.info("--- START: testUnmanagedAppFailPath ---"); @@ -961,6 +968,10 @@ public class TestRMAppTransitions { ArgumentCaptor.forClass(RMAppState.class); verify(writer).applicationFinished(any(RMApp.class), finalState.capture()); Assert.assertEquals(state, finalState.getValue()); + finalState = ArgumentCaptor.forClass(RMAppState.class); + verify(publisher).appFinished(any(RMApp.class), finalState.capture(), + anyLong()); + Assert.assertEquals(state, finalState.getValue()); } private void verifyAppRemovedSchedulerEvent(RMAppState finalState) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index ae318b5..6608ccd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -72,6 +73,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -134,6 +136,7 @@ public class TestRMAppAttemptTransitions { private AMLivelinessMonitor amLivelinessMonitor; private AMLivelinessMonitor amFinishingMonitor; private RMApplicationHistoryWriter writer; + private SystemMetricsPublisher publisher; private RMStateStore store; @@ -246,6 +249,8 @@ public class TestRMAppAttemptTransitions { store = mock(RMStateStore.class); ((RMContextImpl) rmContext).setStateStore(store); + publisher = mock(SystemMetricsPublisher.class); + ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher); scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); @@ -1377,6 +1382,11 @@ public class TestRMAppAttemptTransitions { verify(writer).applicationAttemptFinished( any(RMAppAttempt.class), finalState.capture()); Assert.assertEquals(state, finalState.getValue()); + finalState = + ArgumentCaptor.forClass(RMAppAttemptState.class); + verify(publisher).appAttemptFinished(any(RMAppAttempt.class), finalState.capture(), + anyLong()); + Assert.assertEquals(state, finalState.getValue()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 9862cfe..553587e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; @@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -98,11 +100,13 @@ public class TestRMContainerImpl { Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any()); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); + SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getRMApps()).thenReturn(rmApps); + when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); @@ -111,6 +115,7 @@ public class TestRMContainerImpl { assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); + verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); @@ -143,6 +148,7 @@ public class TestRMContainerImpl { rmContainer.getContainerExitStatus()); assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState()); verify(writer).containerFinished(any(RMContainer.class)); + verify(publisher).containerFinished(any(RMContainer.class), anyLong()); ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor .forClass(RMAppAttemptContainerFinishedEvent.class); @@ -184,10 +190,12 @@ public class TestRMContainerImpl { "host:3465", resource, priority, null); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); + SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); + when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); @@ -196,6 +204,7 @@ public class TestRMContainerImpl { assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); + verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); @@ -224,6 +233,8 @@ public class TestRMContainerImpl { drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); verify(writer, never()).containerFinished(any(RMContainer.class)); + verify(publisher, never()).containerFinished(any(RMContainer.class), + anyLong()); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index fd14ef6..66ec0e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; @@ -249,10 +250,12 @@ public class TestChildQueueOrder { mock(ContainerAllocationExpirer.class); DrainDispatcher drainDispatcher = new DrainDispatcher(); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); + SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); + when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( app_0.getApplicationId(), 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index e548661..fa7145c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; @@ -92,6 +93,7 @@ public class TestUtils { new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), writer); + rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class)); return rmContext; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 3d38364..b4c4c10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -145,6 +146,8 @@ public class TestFifoScheduler { RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, writer); + ((RMContextImpl) rmContext).setSystemMetricsPublisher( + mock(SystemMetricsPublisher.class)); FifoScheduler scheduler = new FifoScheduler(); Configuration conf = new Configuration(); @@ -188,6 +191,8 @@ public class TestFifoScheduler { RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, writer); + ((RMContextImpl) rmContext).setSystemMetricsPublisher( + mock(SystemMetricsPublisher.class)); FifoScheduler scheduler = new FifoScheduler(); scheduler.setRMContext(rmContext); @@ -257,6 +262,8 @@ public class TestFifoScheduler { RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, writer); + ((RMContextImpl) rmContext).setSystemMetricsPublisher( + mock(SystemMetricsPublisher.class)); FifoScheduler scheduler = new FifoScheduler(){ @SuppressWarnings("unused")
