Repository: hadoop Updated Branches: refs/heads/trunk b1fce2b8b -> 4d1f3d902
YARN-6016. Fix minor bugs in handling of local AMRMToken in AMRMProxy. (Botong Huang via Subru). Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d1f3d90 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d1f3d90 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d1f3d90 Branch: refs/heads/trunk Commit: 4d1f3d9020b8a8bf1d2a81e4d6ad20418ed9bcc2 Parents: b1fce2b Author: Subru Krishnan <[email protected]> Authored: Tue Jan 17 14:47:27 2017 -0800 Committer: Subru Krishnan <[email protected]> Committed: Tue Jan 17 14:48:03 2017 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/client/api/impl/TestAMRMProxy.java | 11 ++++++----- .../amrmproxy/AMRMProxyApplicationContextImpl.java | 2 +- .../server/nodemanager/amrmproxy/AMRMProxyService.java | 5 +++++ .../amrmproxy/MockResourceManagerFacade.java | 7 ++++++- .../nodemanager/amrmproxy/TestAMRMProxyService.java | 13 +++++++++++++ 5 files changed, 31 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1f3d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java index 9eef9a0..14df94a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java @@ -139,11 +139,11 @@ public class TestAMRMProxy extends BaseAMRMProxyE2ETest { /* * This test validates the token renewal from the AMRMPRoxy. The test verifies - * that the received token it is different from the previous one within 5 - * requests. + * that the received token from AMRMProxy is different from the previous one + * within 5 requests. */ @Test(timeout = 120000) - public void testE2ETokenRenewal() throws Exception { + public void testAMRMProxyTokenRenewal() throws Exception { ApplicationMasterProtocol client; try (MiniYARNCluster cluster = @@ -176,7 +176,8 @@ public class TestAMRMProxy extends BaseAMRMProxyE2ETest { client.registerApplicationMaster(RegisterApplicationMasterRequest .newInstance(NetUtils.getHostname(), 1024, "")); - LOG.info("testAMRMPRoxy - Allocate Resources Application Master"); + LOG.info( + "testAMRMProxyTokenRenewal - Allocate Resources Application Master"); AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING)); @@ -196,7 +197,7 @@ public class TestAMRMProxy extends BaseAMRMProxyE2ETest { lastToken = response.getAMRMToken(); - // Time slot to be sure the RM renew the token + // Time slot to be sure the AMRMProxy renew the token Thread.sleep(1500); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1f3d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java index 2e5aa94..6d4fdfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java @@ -115,7 +115,7 @@ public class AMRMProxyApplicationContextImpl implements throw new YarnRuntimeException("Missing AMRM token for " + this.applicationAttemptId); } - keyId = this.amrmToken.decodeIdentifier().getKeyId(); + keyId = this.localToken.decodeIdentifier().getKeyId(); this.localTokenKeyId = keyId; } catch (IOException e) { throw new YarnRuntimeException("AMRM token decode error for " http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1f3d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java index dc56090..5e91a20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java @@ -342,9 +342,14 @@ public class AMRMProxyService extends AbstractService implements // check to see if the RM has issued a new AMRMToken & accordingly update // the real ARMRMToken in the current context if (allocateResponse.getAMRMToken() != null) { + LOG.info("RM rolled master-key for amrm-tokens"); + org.apache.hadoop.yarn.api.records.Token token = allocateResponse.getAMRMToken(); + // Do not propagate this info back to AM + allocateResponse.setAMRMToken(null); + org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newTokenId = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>( token.getIdentifier().array(), token.getPassword().array(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1f3d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java index c69313f..f584c94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java @@ -108,6 +108,7 @@ import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.UpdatedContainer; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -297,10 +298,14 @@ public class MockResourceManagerFacade implements Log.getLog().info("Allocating containers: " + containerList.size() + " for application attempt: " + conf.get("AMRMTOKEN")); + + // Always issue a new AMRMToken as if RM rolled master key + Token newAMRMToken = Token.newInstance(new byte[0], "", new byte[0], ""); + return AllocateResponse.newInstance(0, new ArrayList<ContainerStatus>(), containerList, new ArrayList<NodeReport>(), null, AMCommand.AM_RESYNC, 1, null, - new ArrayList<NMToken>(), + new ArrayList<NMToken>(), newAMRMToken, new ArrayList<UpdatedContainer>()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d1f3d90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java index 69b913a..7fffddf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java @@ -401,6 +401,9 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest { AllocateResponse allocateResponse = allocate(appId, allocateRequest); Assert.assertNotNull("allocate() returned null response", allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); containers.addAll(allocateResponse.getAllocatedContainers()); @@ -412,6 +415,9 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest { allocate(appId, Records.newRecord(AllocateRequest.class)); Assert.assertNotNull("allocate() returned null response", allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); containers.addAll(allocateResponse.getAllocatedContainers()); @@ -447,6 +453,9 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest { AllocateResponse allocateResponse = allocate(appId, allocateRequest); Assert.assertNotNull(allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); // The way the mock resource manager is setup, it will return the containers // that were released in the response. This is done because the UAMs run @@ -467,6 +476,10 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest { allocateResponse = allocate(appId, Records.newRecord(AllocateRequest.class)); Assert.assertNotNull(allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); + containersForReleasedContainerIds.addAll(allocateResponse .getAllocatedContainers()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
