HDFS-9276. Failed to Update HDFS Delegation Token for long running application in HA mode. Contributed by Liangliang Gu and John Zhuge
(cherry picked from commit d9aae22fdf2ab22ae8ce4a9d32ac71b3dde084d3) (cherry picked from commit 24d464a15015a7bf6d02d259568fedab1cb97b84) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ad5b278 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ad5b278 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ad5b278 Branch: refs/heads/branch-2.8 Commit: 7ad5b278a71074f2aaf4648dc1a300fc8ab59576 Parents: e216c15 Author: Xiao Chen <[email protected]> Authored: Thu Jul 28 16:29:22 2016 -0700 Committer: Andrew Wang <[email protected]> Committed: Wed Apr 5 16:35:44 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/security/Credentials.java | 22 +++++++++--- .../org/apache/hadoop/security/token/Token.java | 37 +++++++++++++++++--- .../namenode/ha/TestDelegationTokensWithHA.java | 32 +++++++++++++++++ 3 files changed, 83 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ad5b278/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java index 98a72a7..504c9f2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java @@ -91,10 +91,24 @@ public class Credentials implements Writable { * @param t the token object */ public void addToken(Text alias, Token<? extends TokenIdentifier> t) { - if (t != null) { - tokenMap.put(alias, t); - } else { + if (t == null) { LOG.warn("Null token ignored for " + alias); + } else if (tokenMap.put(alias, t) != null) { + // Update private tokens + Map<Text, Token<? extends TokenIdentifier>> tokensToAdd = + new HashMap<>(); + for (Map.Entry<Text, Token<? extends TokenIdentifier>> e : + tokenMap.entrySet()) { + Token<? extends TokenIdentifier> token = e.getValue(); + if (token instanceof Token.PrivateToken && + ((Token.PrivateToken) token).getPublicService().equals(alias)) { + Token<? extends TokenIdentifier> privateToken = + new Token.PrivateToken<>(t); + privateToken.setService(token.getService()); + tokensToAdd.put(e.getKey(), privateToken); + } + } + tokenMap.putAll(tokensToAdd); } } @@ -319,7 +333,7 @@ public class Credentials implements Writable { for(Map.Entry<Text, Token<?>> token: other.tokenMap.entrySet()){ Text key = token.getKey(); if (!tokenMap.containsKey(key) || overwrite) { - tokenMap.put(key, token.getValue()); + addToken(key, token.getValue()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ad5b278/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index e47c585..024b142 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -94,10 +94,10 @@ public class Token<T extends TokenIdentifier> implements Writable { * @param other the token to clone */ public Token(Token<T> other) { - this.identifier = other.identifier; - this.password = other.password; - this.kind = other.kind; - this.service = other.service; + this.identifier = other.identifier.clone(); + this.password = other.password.clone(); + this.kind = new Text(other.kind); + this.service = new Text(other.service); } /** @@ -198,8 +198,37 @@ public class Token<T extends TokenIdentifier> implements Writable { @InterfaceAudience.Private @InterfaceStability.Unstable public static class PrivateToken<T extends TokenIdentifier> extends Token<T> { + final private Text publicService; + public PrivateToken(Token<T> token) { super(token); + publicService = new Text(token.getService()); + } + + public Text getPublicService() { + return publicService; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PrivateToken<?> that = (PrivateToken<?>) o; + return publicService.equals(that.publicService); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + publicService.hashCode(); + return result; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ad5b278/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java index 16b27d8..632bbf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.hdfs.*; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -367,6 +368,37 @@ public class TestDelegationTokensWithHA { token.cancel(conf); } + @Test(timeout = 300000) + public void testCancelAndUpdateDelegationTokens() throws Exception { + // Create UGI with token1 + String user = UserGroupInformation.getCurrentUser().getShortUserName(); + UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser(user); + + ugi1.doAs(new PrivilegedExceptionAction<Void>() { + public Void run() throws Exception { + final Token<DelegationTokenIdentifier> token1 = + getDelegationToken(fs, "JobTracker"); + UserGroupInformation.getCurrentUser() + .addToken(token1.getService(), token1); + + FileSystem fs1 = HATestUtil.configureFailoverFs(cluster, conf); + + // Cancel token1 + doRenewOrCancel(token1, conf, TokenTestAction.CANCEL); + + // Update UGI with token2 + final Token<DelegationTokenIdentifier> token2 = + getDelegationToken(fs, "JobTracker"); + UserGroupInformation.getCurrentUser() + .addToken(token2.getService(), token2); + + // Check whether token2 works + fs1.listFiles(new Path("/"), false); + return null; + } + }); + } + @SuppressWarnings("unchecked") private Token<DelegationTokenIdentifier> getDelegationToken(FileSystem fs, String renewer) throws IOException { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
