[30/49] hadoop git commit: HADOOP-15157. Zookeeper authentication related properties to support CredentialProviders. (Contributed by Gergo Repas)

2018-01-18 Thread curino 
HADOOP-15157. Zookeeper authentication related properties to support 
CredentialProviders. (Contributed by Gergo Repas)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2029353
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2029353
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2029353

Branch: refs/heads/YARN-7402
Commit: b2029353537fc8da9ab67834568cb2e24924cf5a
Parents: 880b9d2
Author: Yufei Gu 
Authored: Tue Jan 16 10:12:10 2018 -0800
Committer: Yufei Gu 
Committed: Tue Jan 16 10:15:22 2018 -0800

--
 .../apache/hadoop/ha/ZKFailoverController.java  | 10 +--
 .../apache/hadoop/security/SecurityUtil.java| 27 ++-
 .../hadoop/util/curator/ZKCuratorManager.java   | 17 +---
 .../src/site/markdown/CredentialProviderAPI.md  |  4 +-
 .../hadoop/security/TestSecurityUtil.java   | 81 
 .../markdown/HDFSHighAvailabilityWithNFS.md |  2 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  2 +-
 7 files changed, 115 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2029353/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index 0ebdacd..a8c19ab 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -22,7 +22,6 @@ import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -341,14 +340,7 @@ public abstract class ZKFailoverController {
 }
 
 // Parse authentication from configuration.
-String zkAuthConf = conf.get(ZK_AUTH_KEY);
-zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf);
-List zkAuths;
-if (zkAuthConf != null) {
-  zkAuths = ZKUtil.parseAuth(zkAuthConf);
-} else {
-  zkAuths = Collections.emptyList();
-}
+List zkAuths = SecurityUtil.getZKAuthInfos(conf, ZK_AUTH_KEY);
 
 // Sanity check configuration.
 Preconditions.checkArgument(zkQuorum != null,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2029353/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 20e8754..5f8cb29 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -28,6 +28,7 @@ import java.net.UnknownHostException;
 import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
@@ -48,7 +49,7 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenInfo;
 import org.apache.hadoop.util.StopWatch;
 import org.apache.hadoop.util.StringUtils;
-
+import org.apache.hadoop.util.ZKUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 //this will need to be replaced someday when there is a suitable replacement
@@ -719,4 +720,28 @@ public final class SecurityUtil {
   public static boolean isPrivilegedPort(final int port) {
 return port < 1024;
   }
+
+  /**
+   * Utility method to fetch ZK auth info from the configuration.
+   * @throws java.io.IOException if the Zookeeper ACLs configuration file
+   * cannot be read
+   * @throws ZKUtil.BadAuthFormatException if the auth format is invalid
+   */
+  public static List getZKAuthInfos(Configuration conf,
+  String configKey) throws IOException {
+char[] zkAuthChars = conf.getPassword(configKey);
+String zkAuthConf =
+zkAuthChars != null ? String.valueOf(zkAuthChars) : null;
+try {
+  zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf);
+  if (zkAuthConf != null) {
+return ZKUtil.parseAuth(zkAuthConf);
+  } else {
+return

hadoop git commit: HADOOP-15157. Zookeeper authentication related properties to support CredentialProviders. (Contributed by Gergo Repas)

2018-01-16 Thread yufei 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 880b9d24f -> b20293535


HADOOP-15157. Zookeeper authentication related properties to support 
CredentialProviders. (Contributed by Gergo Repas)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2029353
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2029353
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2029353

Branch: refs/heads/trunk
Commit: b2029353537fc8da9ab67834568cb2e24924cf5a
Parents: 880b9d2
Author: Yufei Gu 
Authored: Tue Jan 16 10:12:10 2018 -0800
Committer: Yufei Gu 
Committed: Tue Jan 16 10:15:22 2018 -0800

--
 .../apache/hadoop/ha/ZKFailoverController.java  | 10 +--
 .../apache/hadoop/security/SecurityUtil.java| 27 ++-
 .../hadoop/util/curator/ZKCuratorManager.java   | 17 +---
 .../src/site/markdown/CredentialProviderAPI.md  |  4 +-
 .../hadoop/security/TestSecurityUtil.java   | 81 
 .../markdown/HDFSHighAvailabilityWithNFS.md |  2 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  2 +-
 7 files changed, 115 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2029353/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index 0ebdacd..a8c19ab 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -22,7 +22,6 @@ import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -341,14 +340,7 @@ public abstract class ZKFailoverController {
 }
 
 // Parse authentication from configuration.
-String zkAuthConf = conf.get(ZK_AUTH_KEY);
-zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf);
-List zkAuths;
-if (zkAuthConf != null) {
-  zkAuths = ZKUtil.parseAuth(zkAuthConf);
-} else {
-  zkAuths = Collections.emptyList();
-}
+List zkAuths = SecurityUtil.getZKAuthInfos(conf, ZK_AUTH_KEY);
 
 // Sanity check configuration.
 Preconditions.checkArgument(zkQuorum != null,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2029353/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 20e8754..5f8cb29 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -28,6 +28,7 @@ import java.net.UnknownHostException;
 import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
@@ -48,7 +49,7 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenInfo;
 import org.apache.hadoop.util.StopWatch;
 import org.apache.hadoop.util.StringUtils;
-
+import org.apache.hadoop.util.ZKUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 //this will need to be replaced someday when there is a suitable replacement
@@ -719,4 +720,28 @@ public final class SecurityUtil {
   public static boolean isPrivilegedPort(final int port) {
 return port < 1024;
   }
+
+  /**
+   * Utility method to fetch ZK auth info from the configuration.
+   * @throws java.io.IOException if the Zookeeper ACLs configuration file
+   * cannot be read
+   * @throws ZKUtil.BadAuthFormatException if the auth format is invalid
+   */
+  public static List getZKAuthInfos(Configuration conf,
+  String configKey) throws IOException {
+char[] zkAuthChars = conf.getPassword(configKey);
+String zkAuthConf =
+zkAuthChars != null ? String.valueOf(zkAuthChars) : null;
+try {
+  zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf);
+  if (zkAuthConf != null) {
+