Y-5709. Patch v2. WiP
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f05f45f1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f05f45f1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f05f45f1 Branch: refs/heads/yarn-5709 Commit: f05f45f1085ab48cb501bfb82aaa96ff8f4d5b2b Parents: 5ba4836 Author: Karthik Kambatla <ka...@apache.org> Authored: Wed Dec 7 11:02:43 2016 -0800 Committer: Karthik Kambatla <ka...@apache.org> Committed: Wed Dec 7 11:02:43 2016 -0800 ---------------------------------------------------------------------- .../server/resourcemanager/ResourceManager.java | 2 +- .../yarn/server/resourcemanager/MockRM.java | 24 +++++++----- .../server/resourcemanager/RMHATestBase.java | 4 +- .../TestLeaderElectorService.java | 5 +-- .../yarn/server/resourcemanager/TestRMHA.java | 39 ++++++++++---------- 5 files changed, 39 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f05f45f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 192bbca..3f48cc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -278,7 +278,7 @@ public class ResourceManager extends CompositeService implements Recoverable { if (HAUtil.isAutomaticFailoverEnabled(conf) && HAUtil.isAutomaticFailoverEmbedded(conf)) { EmbeddedElector elector = createEmbeddedElector(); - addService(elector); + addIfService(elector); rmContext.setLeaderElectorService(elector); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f05f45f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 1ea2324..755eb54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -136,13 +136,22 @@ public class MockRM extends ResourceManager { } public MockRM(Configuration conf, RMStateStore store) { - this(conf, store, true); + this(conf, store, true, false); } - + + public MockRM(Configuration conf, boolean useRealElector) { + this(conf, null, true, useRealElector); + } + + public MockRM(Configuration conf, RMStateStore store, boolean useRealElector) { + this(conf, store, true, useRealElector); + } + public MockRM(Configuration conf, RMStateStore store, - boolean useNullRMNodeLabelsManager) { + boolean useNullRMNodeLabelsManager, boolean useRealElector) { super(); this.useNullRMNodeLabelsManager = useNullRMNodeLabelsManager; + this.useRealElector = useRealElector; init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf)); if (store != null) { setRMStateStore(store); @@ -163,11 +172,6 @@ public class MockRM extends ResourceManager { disableDrainEventsImplicitly = false; } - public MockRM(Configuration conf, boolean useRealElector) { - this(conf); - this.useRealElector = useRealElector; - } - public class MockRMMemoryStateStore extends MemoryRMStateStore { @SuppressWarnings("rawtypes") @Override @@ -204,9 +208,9 @@ public class MockRM extends ResourceManager { @Override protected EmbeddedElector createEmbeddedElector() throws IOException { if (useRealElector) { - return mock(EmbeddedElector.class); - } else { return super.createEmbeddedElector(); + } else { + return null; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f05f45f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java index 6092f41..c9ce7d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java @@ -108,13 +108,13 @@ public abstract class RMHATestBase extends ClientBaseWithFixes{ } protected void startRMs() throws IOException { - rm1 = new MockRM(confForRM1, null, false){ + rm1 = new MockRM(confForRM1, null, false, false){ @Override protected Dispatcher createDispatcher() { return new DrainDispatcher(); } }; - rm2 = new MockRM(confForRM2, null, false){ + rm2 = new MockRM(confForRM2, null, false, false){ @Override protected Dispatcher createDispatcher() { return new DrainDispatcher(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f05f45f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java index 0c2ee5c..121cacb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestLeaderElectorService.java @@ -63,7 +63,6 @@ public class TestLeaderElectorService { conf = new Configuration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.setBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, true); - conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true); conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1"); conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); @@ -121,7 +120,7 @@ public class TestLeaderElectorService { } }; memStore.init(conf); - rm1 = new MockRM(conf, memStore); + rm1 = new MockRM(conf, memStore, true); rm1.init(conf); rm1.start(); @@ -188,7 +187,7 @@ public class TestLeaderElectorService { Thread launchRM = new Thread() { @Override public void run() { - rm1 = new MockRM(conf) { + rm1 = new MockRM(conf, true) { @Override synchronized void transitionToActive() throws Exception { if (throwException.get()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f05f45f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 000f4a4..5114329 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -161,8 +161,8 @@ public class TestRMHA { ClientResponse response = webResource.path("ws").path("v1").path("cluster").path("apps") - .path(path).accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path(path).accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); @@ -178,13 +178,13 @@ public class TestRMHA { * 1. Standby: Should be a no-op * 2. Active: Active services should start * 3. Active: Should be a no-op. - * While active, submit a couple of jobs + * While active, submit a couple of jobs * 4. Standby: Active services should stop * 5. Active: Active services should start * 6. Stop the RM: All services should stop and RM should not be ready to * become Active */ - @Test (timeout = 30000) + @Test(timeout = 30000) public void testFailoverAndTransitions() throws Exception { configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); Configuration conf = new YarnConfiguration(configuration); @@ -204,37 +204,37 @@ public class TestRMHA { checkMonitorHealth(); checkStandbyRMFunctionality(); verifyClusterMetrics(0, 0, 0, 0, 0, 0); - + // 1. Transition to Standby - must be a no-op rm.adminService.transitionToStandby(requestInfo); checkMonitorHealth(); checkStandbyRMFunctionality(); verifyClusterMetrics(0, 0, 0, 0, 0, 0); - + // 2. Transition to active rm.adminService.transitionToActive(requestInfo); checkMonitorHealth(); checkActiveRMFunctionality(); verifyClusterMetrics(1, 1, 1, 1, 2048, 1); - + // 3. Transition to active - no-op rm.adminService.transitionToActive(requestInfo); checkMonitorHealth(); checkActiveRMFunctionality(); verifyClusterMetrics(1, 2, 2, 2, 2048, 2); - + // 4. Transition to standby rm.adminService.transitionToStandby(requestInfo); checkMonitorHealth(); checkStandbyRMFunctionality(); verifyClusterMetrics(0, 0, 0, 0, 0, 0); - + // 5. Transition to active to check Active->Standby->Active works rm.adminService.transitionToActive(requestInfo); checkMonitorHealth(); checkActiveRMFunctionality(); verifyClusterMetrics(1, 1, 1, 1, 2048, 1); - + // 6. Stop the RM. All services should stop and RM should not be ready to // become active rm.stop(); @@ -340,7 +340,7 @@ public class TestRMHA { rm.adminService.transitionToStandby(requestInfo); rm.adminService.transitionToActive(requestInfo); rm.adminService.transitionToStandby(requestInfo); - + MyCountingDispatcher dispatcher = (MyCountingDispatcher) rm.getRMContext().getDispatcher(); assertTrue(!dispatcher.isStopped()); @@ -348,24 +348,24 @@ public class TestRMHA { rm.adminService.transitionToActive(requestInfo); assertEquals(errorMessageForEventHandler, expectedEventHandlerCount, ((MyCountingDispatcher) rm.getRMContext().getDispatcher()) - .getEventHandlerCount()); + .getEventHandlerCount()); assertEquals(errorMessageForService, expectedServiceCount, rm.getServices().size()); - + // Keep the dispatcher reference before transitioning to standby dispatcher = (MyCountingDispatcher) rm.getRMContext().getDispatcher(); - - + + rm.adminService.transitionToStandby(requestInfo); assertEquals(errorMessageForEventHandler, expectedEventHandlerCount, ((MyCountingDispatcher) rm.getRMContext().getDispatcher()) - .getEventHandlerCount()); + .getEventHandlerCount()); assertEquals(errorMessageForService, expectedServiceCount, rm.getServices().size()); assertTrue(dispatcher.isStopped()); - + rm.stop(); } @@ -386,7 +386,8 @@ public class TestRMHA { assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM1_NODE_ID); //test if RM_HA_ID can not be found - configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID+ "," + RM3_NODE_ID); + configuration + .set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM3_NODE_ID); configuration.unset(YarnConfiguration.RM_HA_ID); conf = new YarnConfiguration(configuration); try { @@ -458,7 +459,7 @@ public class TestRMHA { checkActiveRMFunctionality(); } - @Test(timeout = 90000) + @Test public void testTransitionedToStandbyShouldNotHang() throws Exception { configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); Configuration conf = new YarnConfiguration(configuration); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
