HDFS-6986. DistributedFileSystem must get delegation tokens from configured KeyProvider. (zhz via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b35f816 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b35f816 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b35f816 Branch: refs/heads/HDFS-6584 Commit: 3b35f81603bbfae119762b50bcb46de70a421368 Parents: 0f3c19c Author: Alejandro Abdelnur <t...@apache.org> Authored: Fri Sep 5 22:33:48 2014 -0700 Committer: Alejandro Abdelnur <t...@apache.org> Committed: Fri Sep 5 22:33:48 2014 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++ .../hadoop/hdfs/DistributedFileSystem.java | 24 +++++++++++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 43 ++++++++++++++++++++ 4 files changed, 74 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0772ea6..333bdce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -711,6 +711,9 @@ Release 2.6.0 - UNRELEASED HDFS-6714. TestBlocksScheduledCounter#testBlocksScheduledCounter should shutdown cluster (vinayakumarb) + HDFS-6986. DistributedFileSystem must get delegation tokens from configured + KeyProvider. (zhz via tucu) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 8daf912..e4215f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -3084,4 +3084,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, DFSHedgedReadMetrics getHedgedReadMetrics() { return HEDGED_READ_METRIC; } + + public KeyProviderCryptoExtension getKeyProvider() { + return provider; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index bf7d62e..dbdf5c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -84,8 +84,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -1946,6 +1948,28 @@ public class DistributedFileSystem extends FileSystem { }.resolve(this, absF); } + @Override + public Token<?>[] addDelegationTokens( + final String renewer, Credentials credentials) throws IOException { + Token<?>[] tokens = super.addDelegationTokens(renewer, credentials); + if (dfs.getKeyProvider() != null) { + KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension = + KeyProviderDelegationTokenExtension. + createKeyProviderDelegationTokenExtension(dfs.getKeyProvider()); + Token<?>[] kpTokens = keyProviderDelegationTokenExtension. + addDelegationTokens(renewer, credentials); + if (tokens != null && kpTokens != null) { + Token<?>[] all = new Token<?>[tokens.length + kpTokens.length]; + System.arraycopy(tokens, 0, all, 0, tokens.length); + System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length); + tokens = all; + } else { + tokens = (tokens != null) ? tokens : kpTokens; + } + } + return tokens; + } + public DFSInotifyEventInputStream getInotifyEventStream() throws IOException { return dfs.getInotifyEventStream(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 1a13332..1cf9263 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.apache.hadoop.fs.FSTestWrapper; import org.apache.hadoop.fs.FileContext; @@ -51,12 +52,22 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector; import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.DelegationTokenExtension; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension; +import org.apache.hadoop.io.Text; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import static org.mockito.Mockito.withSettings; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesEqual; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; @@ -91,6 +102,7 @@ public class TestEncryptionZones { conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir + "/test.jks" ); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); // Lower the batch size for testing conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2); @@ -753,4 +765,35 @@ public class TestEncryptionZones { e.getCause()); } } + + /** + * Tests obtaining delegation token from stored key + */ + @Test(timeout = 120000) + public void testDelegationToken() throws Exception { + UserGroupInformation.createRemoteUser("JobTracker"); + DistributedFileSystem dfs = cluster.getFileSystem(); + KeyProviderCryptoExtension keyProvider = Mockito.mock(KeyProviderCryptoExtension.class, + withSettings().extraInterfaces( + DelegationTokenExtension.class, + CryptoExtension.class)); + Mockito.when(keyProvider.getConf()).thenReturn(conf); + byte[] testIdentifier = "Test identifier for delegation token".getBytes(); + + Token<?> testToken = new Token(testIdentifier, new byte[0], + new Text(), new Text()); + Mockito.when(((DelegationTokenExtension)keyProvider). + addDelegationTokens(anyString(), (Credentials)any())). + thenReturn(new Token<?>[] { testToken }); + + dfs.getClient().provider = keyProvider; + + Credentials creds = new Credentials(); + final Token<?> tokens[] = dfs.addDelegationTokens("JobTracker", creds); + DistributedFileSystem.LOG.debug("Delegation tokens: " + + Arrays.asList(tokens)); + Assert.assertEquals(2, tokens.length); + Assert.assertEquals(tokens[1], testToken); + Assert.assertEquals(1, creds.numberOfTokens()); + } }
