Repository: hadoop Updated Branches: refs/heads/branch-3.0 463fcfb50 -> 101d9005d
HADOOP-15390. Yarn RM logs flooded by DelegationTokenRenewer trying to renew KMS tokens (xiaochen via rkanter) (cherry picked from commit 7ab08a9c37a76edbe02d556fcfb2e637f45afc21) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/101d9005 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/101d9005 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/101d9005 Branch: refs/heads/branch-3.0 Commit: 101d9005dd07faeaf18f6e7c69c54d645cd214a9 Parents: 463fcfb Author: Robert Kanter <[email protected]> Authored: Mon Apr 23 15:44:15 2018 -0700 Committer: Robert Kanter <[email protected]> Committed: Mon Apr 23 15:53:53 2018 -0700 ---------------------------------------------------------------------- .../hadoop/crypto/key/kms/KMSTokenRenewer.java | 12 ++++++------ .../security/DelegationTokenRenewer.java | 4 ++++ .../security/TestDelegationTokenRenewer.java | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/101d9005/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java index 908ad39..1fff2f9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java @@ -58,9 +58,9 @@ public class KMSTokenRenewer extends TokenRenewer { try { if (!(keyProvider instanceof KeyProviderDelegationTokenExtension.DelegationTokenExtension)) { - LOG.warn("keyProvider {} cannot renew token {}.", - keyProvider == null ? "null" : keyProvider.getClass(), token); - return 0; + throw new IOException(String + .format("keyProvider %s cannot renew token [%s]", + keyProvider == null ? "null" : keyProvider.getClass(), token)); } return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension) keyProvider).renewDelegationToken(token); @@ -78,9 +78,9 @@ public class KMSTokenRenewer extends TokenRenewer { try { if (!(keyProvider instanceof KeyProviderDelegationTokenExtension.DelegationTokenExtension)) { - LOG.warn("keyProvider {} cannot cancel token {}.", - keyProvider == null ? "null" : keyProvider.getClass(), token); - return; + throw new IOException(String + .format("keyProvider %s cannot cancel token [%s]", + keyProvider == null ? "null" : keyProvider.getClass(), token)); } ((KeyProviderDelegationTokenExtension.DelegationTokenExtension) keyProvider).cancelDelegationToken(token); http://git-wip-us.apache.org/repos/asf/hadoop/blob/101d9005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index abb8d59..220787c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -592,6 +592,10 @@ public class DelegationTokenRenewer extends AbstractService { throws IOException { // calculate timer time long expiresIn = token.expirationDate - System.currentTimeMillis(); + if (expiresIn <= 0) { + LOG.info("Will not renew token " + token); + return; + } long renewIn = token.expirationDate - expiresIn/10; // little bit before the expiration // need to create new task every time RenewalTimerTask tTask = new RenewalTimerTask(token); http://git-wip-us.apache.org/repos/asf/hadoop/blob/101d9005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index c708b92..8351860 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; @@ -1401,4 +1402,22 @@ public class TestDelegationTokenRenewer { .contains(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_CONF_SIZE)); } } + + // Test if the token renewer returned an invalid expiration time, that token's + // renewal should be ignored. + @Test + public void testTokenRenewerInvalidReturn() throws Exception { + DelegationTokenToRenew mockDttr = mock(DelegationTokenToRenew.class); + mockDttr.expirationDate = 0; + delegationTokenRenewer.setTimerForTokenRenewal(mockDttr); + assertNull(mockDttr.timerTask); + + mockDttr.expirationDate = -1; + delegationTokenRenewer.setTimerForTokenRenewal(mockDttr); + assertNull(mockDttr.timerTask); + + mockDttr.expirationDate = System.currentTimeMillis() - 1; + delegationTokenRenewer.setTimerForTokenRenewal(mockDttr); + assertNull(mockDttr.timerTask); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
