Repository: hadoop Updated Branches: refs/heads/branch-2.7 2a5b8e5b3 -> 400a6144e
YARN-5092. TestRMDelegationTokens fails intermittently. Contributed by Jason Lowe. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/400a6144 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/400a6144 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/400a6144 Branch: refs/heads/branch-2.7 Commit: 400a6144e2d9eade9ebe721bdf3ff468f1bc7a91 Parents: 2a5b8e5 Author: Rohith Sharma K S <[email protected]> Authored: Fri Jul 22 20:57:35 2016 +0530 Committer: Rohith Sharma K S <[email protected]> Committed: Fri Jul 22 20:57:35 2016 +0530 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 ++ .../security/TestRMDelegationTokens.java | 38 ++++++++++++-------- 2 files changed, 25 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/400a6144/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a5f0445..fa1f0a6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -23,6 +23,8 @@ Release 2.7.4 - UNRELEASED YARN-3707. RM Web UI queue filter doesn't work. (Wangda Tan via jianhe) + YARN-5092. TestRMDelegationTokens fails intermittently. Contributed by Jason Lowe. + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/400a6144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java index 2847a89..96593d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityM import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -54,24 +53,25 @@ import org.junit.Test; public class TestRMDelegationTokens { - private YarnConfiguration conf; + private YarnConfiguration testConf; @Before public void setup() { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); ExitUtil.disableSystemExit(); - conf = new YarnConfiguration(); - UserGroupInformation.setConfiguration(conf); - conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); - conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); + testConf = new YarnConfiguration(); + UserGroupInformation.setLoginUser(null); + UserGroupInformation.setConfiguration(testConf); + testConf.set(YarnConfiguration.RM_STORE, + MemoryRMStateStore.class.getName()); } // Test the DT mast key in the state-store when the mast key is being rolled. @Test(timeout = 15000) public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); - memStore.init(conf); + memStore.init(testConf); RMState rmState = memStore.getState(); Map<RMDelegationTokenIdentifier, Long> rmDTState = @@ -79,7 +79,7 @@ public class TestRMDelegationTokens { Set<DelegationKey> rmDTMasterKeyState = rmState.getRMDTSecretManagerState().getMasterKeyState(); - MockRM rm1 = new MyMockRM(conf, memStore); + MockRM rm1 = new MyMockRM(testConf, memStore); rm1.start(); // on rm start, two master keys are created. // One is created at RMDTSecretMgr.startThreads.updateCurrentKey(); @@ -90,9 +90,6 @@ public class TestRMDelegationTokens { rm1.getRMContext().getRMDelegationTokenSecretManager(); // assert all master keys are saved Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState); - Set<DelegationKey> expiringKeys = new HashSet<DelegationKey>(); - expiringKeys.addAll(dtSecretManager.getAllMasterKeys()); - // request to generate a RMDelegationToken GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class); @@ -128,13 +125,13 @@ public class TestRMDelegationTokens { @Test(timeout = 15000) public void testRemoveExpiredMasterKeyInRMStateStore() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); - memStore.init(conf); + memStore.init(testConf); RMState rmState = memStore.getState(); Set<DelegationKey> rmDTMasterKeyState = rmState.getRMDTSecretManagerState().getMasterKeyState(); - MockRM rm1 = new MyMockRM(conf, memStore); + MockRM rm1 = new MyMockRM(testConf, memStore); rm1.start(); RMDelegationTokenSecretManager dtSecretManager = rm1.getRMContext().getRMDelegationTokenSecretManager(); @@ -156,6 +153,7 @@ public class TestRMDelegationTokens { break; Thread.sleep(500); } + rm1.stop(); } class MyMockRM extends TestSecurityMockRM { @@ -166,7 +164,7 @@ public class TestRMDelegationTokens { @Override protected RMSecretManagerService createRMSecretManagerService() { - return new RMSecretManagerService(conf, rmContext) { + return new RMSecretManagerService(testConf, rmContext) { @Override protected RMDelegationTokenSecretManager @@ -205,7 +203,17 @@ public class TestRMDelegationTokens { for (int keyId : allKeys.keySet()) { if (keyId == currentId) { DelegationKey currentKey = allKeys.get(keyId); - Assert.assertTrue(rmDTMasterKeyState.contains(currentKey)); + // There's a small window where the key expiry has changed in memory + // but not the state store yet, and DelegationKey hashcode/equals + // uses the expiry so the contains method will fail to find it. + boolean found = false; + for (DelegationKey k : rmDTMasterKeyState) { + if (k.getKeyId() == keyId) { + found = true; + break; + } + } + Assert.assertTrue(found); return currentKey; } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
