subject:"\[2\/2\] hadoop git commit\: HADOOP\-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah."

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

2018-04-10 Thread xiao

HADOOP-14445. Delegation tokens are not shared between KMS instances. 
Contributed by Xiao Chen and Rushabh S Shah.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/714a079f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/714a079f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/714a079f

Branch: refs/heads/branch-2.8
Commit: 714a079ffb88540ec1e09d5023c35e1fa0dd016d
Parents: 5f8ab3a
Author: Xiao Chen 
Authored: Tue Apr 10 15:47:42 2018 -0700
Committer: Xiao Chen 
Committed: Tue Apr 10 15:48:19 2018 -0700

--
 .../crypto/key/kms/KMSClientProvider.java   | 214 
 .../crypto/key/kms/KMSDelegationToken.java  |  22 +-
 .../crypto/key/kms/KMSLegacyTokenRenewer.java   |  56 ++
 .../hadoop/crypto/key/kms/KMSTokenRenewer.java  | 103 
 .../hadoop/crypto/key/kms/package-info.java |  18 +
 .../fs/CommonConfigurationKeysPublic.java   |  10 +
 .../web/DelegationTokenAuthenticatedURL.java|  21 +-
 .../DelegationTokenAuthenticationHandler.java   |   8 +-
 .../web/DelegationTokenAuthenticator.java   |   2 +-
 .../java/org/apache/hadoop/util/KMSUtil.java|  45 +-
 .../hadoop/util/KMSUtilFaultInjector.java   |  49 ++
 ...apache.hadoop.security.token.TokenIdentifier |   1 +
 ...rg.apache.hadoop.security.token.TokenRenewer |   3 +-
 .../src/main/resources/core-default.xml |  20 +
 .../crypto/key/kms/TestKMSClientProvider.java   | 166 ++
 .../kms/TestLoadBalancingKMSClientProvider.java |  67 ++-
 .../org/apache/hadoop/util/TestKMSUtil.java |  65 +++
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 507 ---
 18 files changed, 1176 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/714a079f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index e165ca2..59ec9cc 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -36,11 +36,11 @@ import 
org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.security.token.TokenSelector;
 import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
 import 
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.hadoop.util.HttpExceptionUtils;
-import org.apache.hadoop.util.KMSUtil;
 import org.apache.http.client.utils.URIBuilder;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.slf4j.Logger;
@@ -82,6 +82,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT;
+
 /**
  * KMS client KeyProvider implementation.
  */
@@ -89,16 +92,13 @@ import com.google.common.base.Strings;
 public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 KeyProviderDelegationTokenExtension.DelegationTokenExtension {
 
-  private static final Logger LOG =
+  public static final Logger LOG =
   LoggerFactory.getLogger(KMSClientProvider.class);
 
   private static final String INVALID_SIGNATURE = "Invalid signature";
 
   private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous 
requests are disallowed";
 
-  public static final String TOKEN_KIND_STR = 
KMSDelegationToken.TOKEN_KIND_STR;
-  public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND;
-
   public static final String SCHEME_NAME = "kms";
 
   private static final String UTF8 = "UTF-8";
@@ -123,12 +123,17 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
 
   private final ValueQueue encKeyVersionQueue;
 
+  /* dtService defines the token service value for the kms token.
+   * The value can be legacy format which is ip:port format or it can be uri.
+   * If it's uri

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

2018-04-10 Thread xiao

HADOOP-14445. Delegation tokens are not shared between KMS instances. 
Contributed by Xiao Chen and Rushabh S Shah.

(cherry picked from commit 95cedc5587a495b46748973218454be87ba8b92e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46ac59a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46ac59a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46ac59a9

Branch: refs/heads/branch-2.9
Commit: 46ac59a9bd464da35467ce924f980368a4b5cad4
Parents: 87485d4
Author: Xiao Chen 
Authored: Tue Apr 10 15:46:30 2018 -0700
Committer: Xiao Chen 
Committed: Tue Apr 10 15:48:10 2018 -0700

--
 .../crypto/key/kms/KMSClientProvider.java   | 214 
 .../crypto/key/kms/KMSDelegationToken.java  |  22 +-
 .../crypto/key/kms/KMSLegacyTokenRenewer.java   |  56 ++
 .../hadoop/crypto/key/kms/KMSTokenRenewer.java  | 103 
 .../hadoop/crypto/key/kms/package-info.java |  18 +
 .../fs/CommonConfigurationKeysPublic.java   |  10 +
 .../web/DelegationTokenAuthenticatedURL.java|  21 +-
 .../DelegationTokenAuthenticationHandler.java   |   8 +-
 .../web/DelegationTokenAuthenticator.java   |   2 +-
 .../java/org/apache/hadoop/util/KMSUtil.java|  45 +-
 .../hadoop/util/KMSUtilFaultInjector.java   |  49 ++
 ...apache.hadoop.security.token.TokenIdentifier |   1 +
 ...rg.apache.hadoop.security.token.TokenRenewer |   3 +-
 .../src/main/resources/core-default.xml |  20 +
 .../crypto/key/kms/TestKMSClientProvider.java   | 166 ++
 .../kms/TestLoadBalancingKMSClientProvider.java |  67 ++-
 .../org/apache/hadoop/util/TestKMSUtil.java |  65 +++
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 505 ---
 18 files changed, 1174 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/46ac59a9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index cdd494f..536de53 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -36,11 +36,11 @@ import 
org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.security.token.TokenSelector;
 import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
 import 
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.hadoop.util.HttpExceptionUtils;
-import org.apache.hadoop.util.KMSUtil;
 import org.apache.http.client.utils.URIBuilder;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.map.ObjectWriter;
@@ -83,6 +83,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT;
+
 /**
  * KMS client KeyProvider implementation.
  */
@@ -90,16 +93,13 @@ import com.google.common.base.Strings;
 public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 KeyProviderDelegationTokenExtension.DelegationTokenExtension {
 
-  private static final Logger LOG =
+  public static final Logger LOG =
   LoggerFactory.getLogger(KMSClientProvider.class);
 
   private static final String INVALID_SIGNATURE = "Invalid signature";
 
   private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous 
requests are disallowed";
 
-  public static final String TOKEN_KIND_STR = 
KMSDelegationToken.TOKEN_KIND_STR;
-  public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND;
-
   public static final String SCHEME_NAME = "kms";
 
   private static final String UTF8 = "UTF-8";
@@ -127,12 +127,17 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   private static final ObjectWriter WRITER =
   new ObjectMapper().writerWithDefaultPrettyPrinter();
 
+  /* dtService defines

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

2018-04-10 Thread xiao

HADOOP-14445. Delegation tokens are not shared between KMS instances. 
Contributed by Xiao Chen and Rushabh S Shah.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95cedc55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95cedc55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95cedc55

Branch: refs/heads/branch-2
Commit: 95cedc5587a495b46748973218454be87ba8b92e
Parents: 0fb1457
Author: Xiao Chen 
Authored: Tue Apr 10 15:46:30 2018 -0700
Committer: Xiao Chen 
Committed: Tue Apr 10 15:46:46 2018 -0700

--
 .../crypto/key/kms/KMSClientProvider.java   | 214 
 .../crypto/key/kms/KMSDelegationToken.java  |  22 +-
 .../crypto/key/kms/KMSLegacyTokenRenewer.java   |  56 ++
 .../hadoop/crypto/key/kms/KMSTokenRenewer.java  | 103 
 .../hadoop/crypto/key/kms/package-info.java |  18 +
 .../fs/CommonConfigurationKeysPublic.java   |  10 +
 .../web/DelegationTokenAuthenticatedURL.java|  21 +-
 .../DelegationTokenAuthenticationHandler.java   |   8 +-
 .../web/DelegationTokenAuthenticator.java   |   2 +-
 .../java/org/apache/hadoop/util/KMSUtil.java|  45 +-
 .../hadoop/util/KMSUtilFaultInjector.java   |  49 ++
 ...apache.hadoop.security.token.TokenIdentifier |   1 +
 ...rg.apache.hadoop.security.token.TokenRenewer |   3 +-
 .../src/main/resources/core-default.xml |  20 +
 .../crypto/key/kms/TestKMSClientProvider.java   | 166 ++
 .../kms/TestLoadBalancingKMSClientProvider.java |  67 ++-
 .../org/apache/hadoop/util/TestKMSUtil.java |  65 +++
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 505 ---
 18 files changed, 1174 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/95cedc55/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index cdd494f..536de53 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -36,11 +36,11 @@ import 
org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.security.token.TokenSelector;
 import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
 import 
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.hadoop.util.HttpExceptionUtils;
-import org.apache.hadoop.util.KMSUtil;
 import org.apache.http.client.utils.URIBuilder;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.map.ObjectWriter;
@@ -83,6 +83,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT;
+
 /**
  * KMS client KeyProvider implementation.
  */
@@ -90,16 +93,13 @@ import com.google.common.base.Strings;
 public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 KeyProviderDelegationTokenExtension.DelegationTokenExtension {
 
-  private static final Logger LOG =
+  public static final Logger LOG =
   LoggerFactory.getLogger(KMSClientProvider.class);
 
   private static final String INVALID_SIGNATURE = "Invalid signature";
 
   private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous 
requests are disallowed";
 
-  public static final String TOKEN_KIND_STR = 
KMSDelegationToken.TOKEN_KIND_STR;
-  public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND;
-
   public static final String SCHEME_NAME = "kms";
 
   private static final String UTF8 = "UTF-8";
@@ -127,12 +127,17 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   private static final ObjectWriter WRITER =
   new ObjectMapper().writerWithDefaultPrettyPrinter();
 
+  /* dtService defines the token service value for the kms token.
+   * The value can be

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

2018-04-10 Thread xiao

HADOOP-14445. Delegation tokens are not shared between KMS instances. 
Contributed by Xiao Chen and Rushabh S Shah.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/583fa6ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/583fa6ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/583fa6ed

Branch: refs/heads/trunk
Commit: 583fa6ed48ad3df40bcaa9c591d5ccd07ce3ea81
Parents: e813975
Author: Xiao Chen 
Authored: Tue Apr 10 15:26:33 2018 -0700
Committer: Xiao Chen 
Committed: Tue Apr 10 15:38:25 2018 -0700

--
 .../crypto/key/kms/KMSClientProvider.java   | 212 
 .../crypto/key/kms/KMSDelegationToken.java  |  22 +-
 .../crypto/key/kms/KMSLegacyTokenRenewer.java   |  56 ++
 .../hadoop/crypto/key/kms/KMSTokenRenewer.java  | 103 
 .../hadoop/crypto/key/kms/package-info.java |  18 +
 .../fs/CommonConfigurationKeysPublic.java   |  10 +
 .../web/DelegationTokenAuthenticatedURL.java|  21 +-
 .../DelegationTokenAuthenticationHandler.java   |   8 +-
 .../web/DelegationTokenAuthenticator.java   |   2 +-
 .../java/org/apache/hadoop/util/KMSUtil.java|  45 +-
 .../hadoop/util/KMSUtilFaultInjector.java   |  49 ++
 ...apache.hadoop.security.token.TokenIdentifier |   1 +
 ...rg.apache.hadoop.security.token.TokenRenewer |   3 +-
 .../src/main/resources/core-default.xml |  20 +
 .../crypto/key/kms/TestKMSClientProvider.java   | 162 ++
 .../kms/TestLoadBalancingKMSClientProvider.java |  67 ++-
 .../org/apache/hadoop/util/TestKMSUtil.java |  65 +++
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 519 ---
 18 files changed, 1180 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 2eb2e21..f97fde7 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -36,8 +36,9 @@ import 
org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.security.token.TokenSelector;
 import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
 import 
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.hadoop.util.HttpExceptionUtils;
 import org.apache.hadoop.util.KMSUtil;
@@ -82,6 +83,8 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT;
 import static org.apache.hadoop.util.KMSUtil.checkNotEmpty;
 import static org.apache.hadoop.util.KMSUtil.checkNotNull;
 import static org.apache.hadoop.util.KMSUtil.parseJSONEncKeyVersion;
@@ -96,16 +99,13 @@ import static 
org.apache.hadoop.util.KMSUtil.parseJSONMetadata;
 public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 KeyProviderDelegationTokenExtension.DelegationTokenExtension {
 
-  private static final Logger LOG =
+  public static final Logger LOG =
   LoggerFactory.getLogger(KMSClientProvider.class);
 
   private static final String INVALID_SIGNATURE = "Invalid signature";
 
   private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous 
requests are disallowed";
 
-  public static final String TOKEN_KIND_STR = 
KMSDelegationToken.TOKEN_KIND_STR;
-  public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND;
-
   public static final String SCHEME_NAME = "kms";
 
   private static final String UTF8 = "UTF-8";
@@ -133,12 +133,17 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   private static final ObjectWriter WRITER =
   new ObjectMapper().writerWithDefaultPrettyPrinter();
 
+  /* dtService defines the token service value for the kms token.
+   * The

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

[2/2] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.

4 matches

Site Navigation

Mail list logo

Footer information