[1/2] hadoop git commit: HADOOP-14445. Use DelegationTokenIssuer to create KMS delegation tokens that can authenticate to all KMS instances. Contributed by Daryn Sharp, Xiao Chen, Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 53b522af6 -> ff7ca472d
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff7ca472/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
index de27f7e..30e8aa7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
@@ -35,14 +35,12 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.KMSUtil;
/**
@@ -71,32 +69,6 @@ public final class HdfsKMSUtil {
return KMSUtil.createKeyProvider(conf, keyProviderUriKeyName);
}
- public static Token[] addDelegationTokensForKeyProvider(
- KeyProviderTokenIssuer kpTokenIssuer, final String renewer,
- Credentials credentials, URI namenodeUri, Token[] tokens)
- throws IOException {
-KeyProvider keyProvider = kpTokenIssuer.getKeyProvider();
-if (keyProvider != null) {
- KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension
- = KeyProviderDelegationTokenExtension.
- createKeyProviderDelegationTokenExtension(keyProvider);
- Token[] kpTokens = keyProviderDelegationTokenExtension.
- addDelegationTokens(renewer, credentials);
- credentials.addSecretKey(getKeyProviderMapKey(namenodeUri),
- DFSUtilClient.string2Bytes(
- kpTokenIssuer.getKeyProviderUri().toString()));
- if (tokens != null && kpTokens != null) {
-Token[] all = new Token[tokens.length + kpTokens.length];
-System.arraycopy(tokens, 0, all, 0, tokens.length);
-System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
-tokens = all;
- } else {
-tokens = (tokens != null) ? tokens : kpTokens;
- }
-}
-return tokens;
- }
-
/**
* Obtain the crypto protocol version from the provided FileEncryptionInfo,
* checking to see if this version is supported by.
@@ -161,28 +133,36 @@ public final class HdfsKMSUtil {
URI keyProviderUri = null;
// Lookup the secret in credentials object for namenodeuri.
Credentials credentials = ugi.getCredentials();
+Text credsKey = getKeyProviderMapKey(namenodeUri);
byte[] keyProviderUriBytes =
-credentials.getSecretKey(getKeyProviderMapKey(namenodeUri));
+credentials.getSecretKey(credsKey);
if(keyProviderUriBytes != null) {
keyProviderUri =
URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes));
- return keyProviderUri;
}
-
-if (keyProviderUriStr != null) {
- if (!keyProviderUriStr.isEmpty()) {
+if (keyProviderUri == null) {
+ // NN is old and doesn't report provider, so use conf.
+ if (keyProviderUriStr == null) {
+keyProviderUri = KMSUtil.getKeyProviderUri(conf,
keyProviderUriKeyName);
+ } else if (!keyProviderUriStr.isEmpty()) {
keyProviderUri = URI.create(keyProviderUriStr);
}
- return keyProviderUri;
+ if (keyProviderUri != null) {
+credentials.addSecretKey(
+credsKey, DFSUtilClient.string2Bytes(keyProviderUri.toString()));
+ }
}
+return keyProviderUri;
+ }
-// Last thing is to trust its own conf to be backwards compatible.
-String keyProviderUriFromConf = conf.getTrimmed(
-CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
-if (keyProviderUriFromConf != null && !keyProviderUriFromConf.isEmpty()) {
- keyProviderUri = URI.create(keyProviderUriFromConf);
+ public static KeyProvider getKeyProvider(KeyProviderTokenIssuer issuer,
+ Configuration conf)
+ throws IOException {
+URI keyProviderUri = issuer.getKeyProviderUri();
+if (keyProviderUri != null) {
+ return KMSUtil.createKeyProviderFromUri(conf, keyProviderUri);
}
-return keyProviderUri;
+return null;
}
/**
[1/2] hadoop git commit: HADOOP-14445. Use DelegationTokenIssuer to create KMS delegation tokens that can authenticate to all KMS instances. Contributed by Daryn Sharp, Xiao Chen, Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 65c1469b1 -> 9cb0654fb
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9cb0654f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
index de27f7e..30e8aa7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
@@ -35,14 +35,12 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.KMSUtil;
/**
@@ -71,32 +69,6 @@ public final class HdfsKMSUtil {
return KMSUtil.createKeyProvider(conf, keyProviderUriKeyName);
}
- public static Token[] addDelegationTokensForKeyProvider(
- KeyProviderTokenIssuer kpTokenIssuer, final String renewer,
- Credentials credentials, URI namenodeUri, Token[] tokens)
- throws IOException {
-KeyProvider keyProvider = kpTokenIssuer.getKeyProvider();
-if (keyProvider != null) {
- KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension
- = KeyProviderDelegationTokenExtension.
- createKeyProviderDelegationTokenExtension(keyProvider);
- Token[] kpTokens = keyProviderDelegationTokenExtension.
- addDelegationTokens(renewer, credentials);
- credentials.addSecretKey(getKeyProviderMapKey(namenodeUri),
- DFSUtilClient.string2Bytes(
- kpTokenIssuer.getKeyProviderUri().toString()));
- if (tokens != null && kpTokens != null) {
-Token[] all = new Token[tokens.length + kpTokens.length];
-System.arraycopy(tokens, 0, all, 0, tokens.length);
-System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
-tokens = all;
- } else {
-tokens = (tokens != null) ? tokens : kpTokens;
- }
-}
-return tokens;
- }
-
/**
* Obtain the crypto protocol version from the provided FileEncryptionInfo,
* checking to see if this version is supported by.
@@ -161,28 +133,36 @@ public final class HdfsKMSUtil {
URI keyProviderUri = null;
// Lookup the secret in credentials object for namenodeuri.
Credentials credentials = ugi.getCredentials();
+Text credsKey = getKeyProviderMapKey(namenodeUri);
byte[] keyProviderUriBytes =
-credentials.getSecretKey(getKeyProviderMapKey(namenodeUri));
+credentials.getSecretKey(credsKey);
if(keyProviderUriBytes != null) {
keyProviderUri =
URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes));
- return keyProviderUri;
}
-
-if (keyProviderUriStr != null) {
- if (!keyProviderUriStr.isEmpty()) {
+if (keyProviderUri == null) {
+ // NN is old and doesn't report provider, so use conf.
+ if (keyProviderUriStr == null) {
+keyProviderUri = KMSUtil.getKeyProviderUri(conf,
keyProviderUriKeyName);
+ } else if (!keyProviderUriStr.isEmpty()) {
keyProviderUri = URI.create(keyProviderUriStr);
}
- return keyProviderUri;
+ if (keyProviderUri != null) {
+credentials.addSecretKey(
+credsKey, DFSUtilClient.string2Bytes(keyProviderUri.toString()));
+ }
}
+return keyProviderUri;
+ }
-// Last thing is to trust its own conf to be backwards compatible.
-String keyProviderUriFromConf = conf.getTrimmed(
-CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
-if (keyProviderUriFromConf != null && !keyProviderUriFromConf.isEmpty()) {
- keyProviderUri = URI.create(keyProviderUriFromConf);
+ public static KeyProvider getKeyProvider(KeyProviderTokenIssuer issuer,
+ Configuration conf)
+ throws IOException {
+URI keyProviderUri = issuer.getKeyProviderUri();
+if (keyProviderUri != null) {
+ return KMSUtil.createKeyProviderFromUri(conf, keyProviderUri);
}
-return keyProviderUri;
+return null;
}
/**
[1/2] hadoop git commit: HADOOP-14445. Use DelegationTokenIssuer to create KMS delegation tokens that can authenticate to all KMS instances. Contributed by Daryn Sharp, Xiao Chen, Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 6342a7cb9 -> 6a1ce74fb
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a1ce74f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
index de27f7e..30e8aa7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
@@ -35,14 +35,12 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.KMSUtil;
/**
@@ -71,32 +69,6 @@ public final class HdfsKMSUtil {
return KMSUtil.createKeyProvider(conf, keyProviderUriKeyName);
}
- public static Token[] addDelegationTokensForKeyProvider(
- KeyProviderTokenIssuer kpTokenIssuer, final String renewer,
- Credentials credentials, URI namenodeUri, Token[] tokens)
- throws IOException {
-KeyProvider keyProvider = kpTokenIssuer.getKeyProvider();
-if (keyProvider != null) {
- KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension
- = KeyProviderDelegationTokenExtension.
- createKeyProviderDelegationTokenExtension(keyProvider);
- Token[] kpTokens = keyProviderDelegationTokenExtension.
- addDelegationTokens(renewer, credentials);
- credentials.addSecretKey(getKeyProviderMapKey(namenodeUri),
- DFSUtilClient.string2Bytes(
- kpTokenIssuer.getKeyProviderUri().toString()));
- if (tokens != null && kpTokens != null) {
-Token[] all = new Token[tokens.length + kpTokens.length];
-System.arraycopy(tokens, 0, all, 0, tokens.length);
-System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
-tokens = all;
- } else {
-tokens = (tokens != null) ? tokens : kpTokens;
- }
-}
-return tokens;
- }
-
/**
* Obtain the crypto protocol version from the provided FileEncryptionInfo,
* checking to see if this version is supported by.
@@ -161,28 +133,36 @@ public final class HdfsKMSUtil {
URI keyProviderUri = null;
// Lookup the secret in credentials object for namenodeuri.
Credentials credentials = ugi.getCredentials();
+Text credsKey = getKeyProviderMapKey(namenodeUri);
byte[] keyProviderUriBytes =
-credentials.getSecretKey(getKeyProviderMapKey(namenodeUri));
+credentials.getSecretKey(credsKey);
if(keyProviderUriBytes != null) {
keyProviderUri =
URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes));
- return keyProviderUri;
}
-
-if (keyProviderUriStr != null) {
- if (!keyProviderUriStr.isEmpty()) {
+if (keyProviderUri == null) {
+ // NN is old and doesn't report provider, so use conf.
+ if (keyProviderUriStr == null) {
+keyProviderUri = KMSUtil.getKeyProviderUri(conf,
keyProviderUriKeyName);
+ } else if (!keyProviderUriStr.isEmpty()) {
keyProviderUri = URI.create(keyProviderUriStr);
}
- return keyProviderUri;
+ if (keyProviderUri != null) {
+credentials.addSecretKey(
+credsKey, DFSUtilClient.string2Bytes(keyProviderUri.toString()));
+ }
}
+return keyProviderUri;
+ }
-// Last thing is to trust its own conf to be backwards compatible.
-String keyProviderUriFromConf = conf.getTrimmed(
-CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
-if (keyProviderUriFromConf != null && !keyProviderUriFromConf.isEmpty()) {
- keyProviderUri = URI.create(keyProviderUriFromConf);
+ public static KeyProvider getKeyProvider(KeyProviderTokenIssuer issuer,
+ Configuration conf)
+ throws IOException {
+URI keyProviderUri = issuer.getKeyProviderUri();
+if (keyProviderUri != null) {
+ return KMSUtil.createKeyProviderFromUri(conf, keyProviderUri);
}
-return keyProviderUri;
+return null;
}
/**
[1/2] hadoop git commit: HADOOP-14445. Use DelegationTokenIssuer to create KMS delegation tokens that can authenticate to all KMS instances. Contributed by Daryn Sharp, Xiao Chen, Rushabh S Shah.
Repository: hadoop
Updated Branches:
refs/heads/trunk 6e0e6daaf -> 5ec86b445
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec86b44/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
index de27f7e..30e8aa7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java
@@ -35,14 +35,12 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.KMSUtil;
/**
@@ -71,32 +69,6 @@ public final class HdfsKMSUtil {
return KMSUtil.createKeyProvider(conf, keyProviderUriKeyName);
}
- public static Token[] addDelegationTokensForKeyProvider(
- KeyProviderTokenIssuer kpTokenIssuer, final String renewer,
- Credentials credentials, URI namenodeUri, Token[] tokens)
- throws IOException {
-KeyProvider keyProvider = kpTokenIssuer.getKeyProvider();
-if (keyProvider != null) {
- KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension
- = KeyProviderDelegationTokenExtension.
- createKeyProviderDelegationTokenExtension(keyProvider);
- Token[] kpTokens = keyProviderDelegationTokenExtension.
- addDelegationTokens(renewer, credentials);
- credentials.addSecretKey(getKeyProviderMapKey(namenodeUri),
- DFSUtilClient.string2Bytes(
- kpTokenIssuer.getKeyProviderUri().toString()));
- if (tokens != null && kpTokens != null) {
-Token[] all = new Token[tokens.length + kpTokens.length];
-System.arraycopy(tokens, 0, all, 0, tokens.length);
-System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
-tokens = all;
- } else {
-tokens = (tokens != null) ? tokens : kpTokens;
- }
-}
-return tokens;
- }
-
/**
* Obtain the crypto protocol version from the provided FileEncryptionInfo,
* checking to see if this version is supported by.
@@ -161,28 +133,36 @@ public final class HdfsKMSUtil {
URI keyProviderUri = null;
// Lookup the secret in credentials object for namenodeuri.
Credentials credentials = ugi.getCredentials();
+Text credsKey = getKeyProviderMapKey(namenodeUri);
byte[] keyProviderUriBytes =
-credentials.getSecretKey(getKeyProviderMapKey(namenodeUri));
+credentials.getSecretKey(credsKey);
if(keyProviderUriBytes != null) {
keyProviderUri =
URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes));
- return keyProviderUri;
}
-
-if (keyProviderUriStr != null) {
- if (!keyProviderUriStr.isEmpty()) {
+if (keyProviderUri == null) {
+ // NN is old and doesn't report provider, so use conf.
+ if (keyProviderUriStr == null) {
+keyProviderUri = KMSUtil.getKeyProviderUri(conf,
keyProviderUriKeyName);
+ } else if (!keyProviderUriStr.isEmpty()) {
keyProviderUri = URI.create(keyProviderUriStr);
}
- return keyProviderUri;
+ if (keyProviderUri != null) {
+credentials.addSecretKey(
+credsKey, DFSUtilClient.string2Bytes(keyProviderUri.toString()));
+ }
}
+return keyProviderUri;
+ }
-// Last thing is to trust its own conf to be backwards compatible.
-String keyProviderUriFromConf = conf.getTrimmed(
-CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
-if (keyProviderUriFromConf != null && !keyProviderUriFromConf.isEmpty()) {
- keyProviderUri = URI.create(keyProviderUriFromConf);
+ public static KeyProvider getKeyProvider(KeyProviderTokenIssuer issuer,
+ Configuration conf)
+ throws IOException {
+URI keyProviderUri = issuer.getKeyProviderUri();
+if (keyProviderUri != null) {
+ return KMSUtil.createKeyProviderFromUri(conf, keyProviderUri);
}
-return keyProviderUri;
+return null;
}
/**
