HDFS-9276. Failed to Update HDFS Delegation Token for long running application in HA mode. Contributed by Liangliang Gu and John Zhuge
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9aae22f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9aae22f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9aae22f Branch: refs/heads/HADOOP-12756 Commit: d9aae22fdf2ab22ae8ce4a9d32ac71b3dde084d3 Parents: ce3d68e Author: Xiao Chen <x...@apache.org> Authored: Thu Jul 28 16:29:22 2016 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Thu Jul 28 16:29:40 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/security/Credentials.java | 22 +++++++++--- .../org/apache/hadoop/security/token/Token.java | 37 +++++++++++++++++--- .../namenode/ha/TestDelegationTokensWithHA.java | 32 +++++++++++++++++ 3 files changed, 83 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9aae22f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java index 977ccb5..c62a49c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java @@ -95,10 +95,24 @@ public class Credentials implements Writable { * @param t the token object */ public void addToken(Text alias, Token<? extends TokenIdentifier> t) { - if (t != null) { - tokenMap.put(alias, t); - } else { + if (t == null) { LOG.warn("Null token ignored for " + alias); + } else if (tokenMap.put(alias, t) != null) { + // Update private tokens + Map<Text, Token<? extends TokenIdentifier>> tokensToAdd = + new HashMap<>(); + for (Map.Entry<Text, Token<? extends TokenIdentifier>> e : + tokenMap.entrySet()) { + Token<? extends TokenIdentifier> token = e.getValue(); + if (token instanceof Token.PrivateToken && + ((Token.PrivateToken) token).getPublicService().equals(alias)) { + Token<? extends TokenIdentifier> privateToken = + new Token.PrivateToken<>(t); + privateToken.setService(token.getService()); + tokensToAdd.put(e.getKey(), privateToken); + } + } + tokenMap.putAll(tokensToAdd); } } @@ -397,7 +411,7 @@ public class Credentials implements Writable { for(Map.Entry<Text, Token<?>> token: other.tokenMap.entrySet()){ Text key = token.getKey(); if (!tokenMap.containsKey(key) || overwrite) { - tokenMap.put(key, token.getValue()); + addToken(key, token.getValue()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9aae22f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index 4ea686e..784e797 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -96,10 +96,10 @@ public class Token<T extends TokenIdentifier> implements Writable { * @param other the token to clone */ public Token(Token<T> other) { - this.identifier = other.identifier; - this.password = other.password; - this.kind = other.kind; - this.service = other.service; + this.identifier = other.identifier.clone(); + this.password = other.password.clone(); + this.kind = new Text(other.kind); + this.service = new Text(other.service); } public Token<T> copyToken() { @@ -230,8 +230,37 @@ public class Token<T extends TokenIdentifier> implements Writable { @InterfaceAudience.Private @InterfaceStability.Unstable public static class PrivateToken<T extends TokenIdentifier> extends Token<T> { + final private Text publicService; + public PrivateToken(Token<T> token) { super(token); + publicService = new Text(token.getService()); + } + + public Text getPublicService() { + return publicService; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PrivateToken<?> that = (PrivateToken<?>) o; + return publicService.equals(that.publicService); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + publicService.hashCode(); + return result; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9aae22f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java index 16b27d8..632bbf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.hdfs.*; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -367,6 +368,37 @@ public class TestDelegationTokensWithHA { token.cancel(conf); } + @Test(timeout = 300000) + public void testCancelAndUpdateDelegationTokens() throws Exception { + // Create UGI with token1 + String user = UserGroupInformation.getCurrentUser().getShortUserName(); + UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser(user); + + ugi1.doAs(new PrivilegedExceptionAction<Void>() { + public Void run() throws Exception { + final Token<DelegationTokenIdentifier> token1 = + getDelegationToken(fs, "JobTracker"); + UserGroupInformation.getCurrentUser() + .addToken(token1.getService(), token1); + + FileSystem fs1 = HATestUtil.configureFailoverFs(cluster, conf); + + // Cancel token1 + doRenewOrCancel(token1, conf, TokenTestAction.CANCEL); + + // Update UGI with token2 + final Token<DelegationTokenIdentifier> token2 = + getDelegationToken(fs, "JobTracker"); + UserGroupInformation.getCurrentUser() + .addToken(token2.getService(), token2); + + // Check whether token2 works + fs1.listFiles(new Path("/"), false); + return null; + } + }); + } + @SuppressWarnings("unchecked") private Token<DelegationTokenIdentifier> getDelegationToken(FileSystem fs, String renewer) throws IOException { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
