[01/50] [abbrv] hadoop git commit: HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi.

2018-08-25 Thread shv 
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-12943 90e61bf0f -> 191faeb96


HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edeb2a35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edeb2a35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edeb2a35

Branch: refs/heads/HDFS-12943
Commit: edeb2a356ad671d962764c6e2aee9f9e7d6f394c
Parents: 2d13e41
Author: Xiao Chen 
Authored: Thu Aug 16 22:32:32 2018 -0700
Committer: Xiao Chen 
Committed: Thu Aug 16 22:42:03 2018 -0700

--
 .../key/kms/LoadBalancingKMSClientProvider.java |  43 ++---
 .../kms/TestLoadBalancingKMSClientProvider.java | 181 ++-
 2 files changed, 193 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/edeb2a35/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
index 23cdc50..e68e844 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -113,8 +113,8 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 return providers;
   }
 
-  private  T doOp(ProviderCallable op, int currPos)
-  throws IOException {
+  private  T doOp(ProviderCallable op, int currPos,
+  boolean isIdempotent) throws IOException {
 if (providers.length == 0) {
   throw new IOException("No providers configured !");
 }
@@ -143,7 +143,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 }
 RetryAction action = null;
 try {
-  action = retryPolicy.shouldRetry(ioe, 0, numFailovers, false);
+  action = retryPolicy.shouldRetry(ioe, 0, numFailovers, isIdempotent);
 } catch (Exception e) {
   if (e instanceof IOException) {
 throw (IOException)e;
@@ -201,7 +201,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   public Token[] call(KMSClientProvider provider) throws IOException {
 return provider.addDelegationTokens(renewer, credentials);
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   @Override
@@ -211,7 +211,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   public Long call(KMSClientProvider provider) throws IOException {
 return provider.renewDelegationToken(token);
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   @Override
@@ -222,7 +222,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 provider.cancelDelegationToken(token);
 return null;
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   // This request is sent to all providers in the load-balancing group
@@ -275,7 +275,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.generateEncryptedKey(encryptionKeyName);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -295,7 +295,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.decryptEncryptedKey(encryptedKeyVersion);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -315,7 +315,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.reencryptEncryptedKey(ekv);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -335,7 +335,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   provider.reencryptEncryptedKeys(ekvs);
   retu

[13/14] hadoop git commit: HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi.

2018-08-20 Thread arp 
HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a630a27c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a630a27c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a630a27c

Branch: refs/heads/branch-3.1
Commit: a630a27c53107322a72f9b76e395c4537b09c3fc
Parents: 8656500
Author: Xiao Chen 
Authored: Thu Aug 16 22:32:32 2018 -0700
Committer: Arpit Agarwal 
Committed: Mon Aug 20 14:57:51 2018 -0700

--
 .../key/kms/LoadBalancingKMSClientProvider.java |  43 ++---
 .../kms/TestLoadBalancingKMSClientProvider.java | 181 ++-
 2 files changed, 193 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a630a27c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
index 9677b0d..e0ffdb1 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -113,8 +113,8 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 return providers;
   }
 
-  private  T doOp(ProviderCallable op, int currPos)
-  throws IOException {
+  private  T doOp(ProviderCallable op, int currPos,
+  boolean isIdempotent) throws IOException {
 if (providers.length == 0) {
   throw new IOException("No providers configured !");
 }
@@ -143,7 +143,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 }
 RetryAction action = null;
 try {
-  action = retryPolicy.shouldRetry(ioe, 0, numFailovers, false);
+  action = retryPolicy.shouldRetry(ioe, 0, numFailovers, isIdempotent);
 } catch (Exception e) {
   if (e instanceof IOException) {
 throw (IOException)e;
@@ -201,7 +201,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   public Token[] call(KMSClientProvider provider) throws IOException {
 return provider.addDelegationTokens(renewer, credentials);
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   @Override
@@ -211,7 +211,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   public Long call(KMSClientProvider provider) throws IOException {
 return provider.renewDelegationToken(token);
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   @Override
@@ -222,7 +222,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 provider.cancelDelegationToken(token);
 return null;
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   // This request is sent to all providers in the load-balancing group
@@ -275,7 +275,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.generateEncryptedKey(encryptionKeyName);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -295,7 +295,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.decryptEncryptedKey(encryptedKeyVersion);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -315,7 +315,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.reencryptEncryptedKey(ekv);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -335,7 +335,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   provider.reencryptEncryptedKeys(ekvs);
   return null;
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch

hadoop git commit: HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi.

2018-08-16 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 2d13e410d -> edeb2a356


HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edeb2a35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edeb2a35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edeb2a35

Branch: refs/heads/trunk
Commit: edeb2a356ad671d962764c6e2aee9f9e7d6f394c
Parents: 2d13e41
Author: Xiao Chen 
Authored: Thu Aug 16 22:32:32 2018 -0700
Committer: Xiao Chen 
Committed: Thu Aug 16 22:42:03 2018 -0700

--
 .../key/kms/LoadBalancingKMSClientProvider.java |  43 ++---
 .../kms/TestLoadBalancingKMSClientProvider.java | 181 ++-
 2 files changed, 193 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/edeb2a35/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
index 23cdc50..e68e844 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -113,8 +113,8 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 return providers;
   }
 
-  private  T doOp(ProviderCallable op, int currPos)
-  throws IOException {
+  private  T doOp(ProviderCallable op, int currPos,
+  boolean isIdempotent) throws IOException {
 if (providers.length == 0) {
   throw new IOException("No providers configured !");
 }
@@ -143,7 +143,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 }
 RetryAction action = null;
 try {
-  action = retryPolicy.shouldRetry(ioe, 0, numFailovers, false);
+  action = retryPolicy.shouldRetry(ioe, 0, numFailovers, isIdempotent);
 } catch (Exception e) {
   if (e instanceof IOException) {
 throw (IOException)e;
@@ -201,7 +201,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   public Token[] call(KMSClientProvider provider) throws IOException {
 return provider.addDelegationTokens(renewer, credentials);
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   @Override
@@ -211,7 +211,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   public Long call(KMSClientProvider provider) throws IOException {
 return provider.renewDelegationToken(token);
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   @Override
@@ -222,7 +222,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 provider.cancelDelegationToken(token);
 return null;
   }
-}, nextIdx());
+}, nextIdx(), false);
   }
 
   // This request is sent to all providers in the load-balancing group
@@ -275,7 +275,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.generateEncryptedKey(encryptionKeyName);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -295,7 +295,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.decryptEncryptedKey(encryptedKeyVersion);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -315,7 +315,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
 throws IOException, GeneralSecurityException {
   return provider.reencryptEncryptedKey(ekv);
 }
-  }, nextIdx());
+  }, nextIdx(), true);
 } catch (WrapperException we) {
   if (we.getCause() instanceof GeneralSecurityException) {
 throw (GeneralSecurityException) we.getCause();
@@ -335,7 +335,7 @@ public class LoadBalancingKMSClientProvider extends 
KeyProvider implements
   provider.reencryptEncryptedKeys(ekvs);
   return null;