This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new be508b8 HIVE-22195: Configure authentication type for Zookeeper when
different from the default cluster wide (Denys Kuzmenko via Peter Vary)
be508b8 is described below
commit be508b8d16c465ac17f3b8c09d7cbb27c4eee47a
Author: denys kuzmenko <[email protected]>
AuthorDate: Mon Sep 23 11:04:08 2019 +0200
HIVE-22195: Configure authentication type for Zookeeper when different from
the default cluster wide (Denys Kuzmenko via Peter Vary)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 8 ++-
.../hadoop/hive/registry/impl/ZkRegistryBase.java | 13 +++--
.../hadoop/hive/registry/impl/ZookeeperUtils.java | 18 +++++-
.../hive/registry/impl/TestZookeeperUtils.java | 66 ++++++++++++++++++++++
.../server/HS2ActivePassiveHARegistryClient.java | 2 +-
.../apache/hive/service/server/HiveServer2.java | 50 ++++++++--------
.../metastore/security/ZooKeeperTokenStore.java | 4 +-
7 files changed, 129 insertions(+), 32 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 97cffff..de28e92 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2582,7 +2582,13 @@ public class HiveConf extends Configuration {
"this is set to false, however unless MAPREDUCE-7086 fix is present,
queries that\n" +
"read MM tables with original files will fail. The default in Hive 3.0
is false."),
- // Zookeeper related configs
+ // Zookeeper related configs
+
HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION("hive.security.zookeeper.authentication",
+ "DEFAULT", new StringSet("DEFAULT", "SIMPLE"),
+ "Whether the authentication type for Zookeeper is different from the
cluster wide\n" +
+ "`hadoop.security.authentication` configuration. This could be useful
when cluster\n" +
+ "is kerberized, but Zookeeper is not."),
+
HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "",
"List of ZooKeeper servers to talk to. This is needed for: \n" +
"1. Read/write locks - when hive.lock.manager is set to \n" +
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
index e56ae11..5751b8e 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
@@ -160,7 +160,7 @@ public abstract class ZkRegistryBase<InstanceType extends
ServiceInstance> {
this.stateChangeListeners = new HashSet<>();
this.pathToInstanceCache = new ConcurrentHashMap<>();
this.nodeToInstanceCache = new ConcurrentHashMap<>();
- final String namespace = getRootNamespace(rootNs, nsPrefix);
+ final String namespace = getRootNamespace(conf, rootNs, nsPrefix);
ACLProvider aclProvider;
// get acl provider for most outer path that is non-null
if (userPathPrefix == null) {
@@ -180,8 +180,9 @@ public abstract class ZkRegistryBase<InstanceType extends
ServiceInstance> {
this.zooKeeperClient.getConnectionStateListenable().addListener(new
ZkConnectionStateListener());
}
- public static String getRootNamespace(String userProvidedNamespace, String
defaultNamespacePrefix) {
- final boolean isSecure = UserGroupInformation.isSecurityEnabled();
+ public static String getRootNamespace(Configuration conf, String
userProvidedNamespace,
+ String defaultNamespacePrefix) {
+ final boolean isSecure = ZookeeperUtils.isKerberosEnabled(conf);
String rootNs = userProvidedNamespace;
if (rootNs == null) {
rootNs = defaultNamespacePrefix + (isSecure ? SASL_NAMESPACE :
UNSECURE_NAMESPACE);
@@ -190,7 +191,7 @@ public abstract class ZkRegistryBase<InstanceType extends
ServiceInstance> {
}
private ACLProvider getACLProviderForZKPath(String zkPath) {
- final boolean isSecure = UserGroupInformation.isSecurityEnabled();
+ final boolean isSecure = ZookeeperUtils.isKerberosEnabled(conf);
return new ACLProvider() {
@Override
public List<ACL> getDefaultAcl() {
@@ -366,7 +367,9 @@ public abstract class ZkRegistryBase<InstanceType extends
ServiceInstance> {
}
private void checkAndSetAcls() throws Exception {
- if (!UserGroupInformation.isSecurityEnabled()) return;
+ if (!ZookeeperUtils.isKerberosEnabled(conf)) {
+ return;
+ }
// We are trying to check ACLs on the "workers" directory, which noone
except us should be
// able to write to. Higher-level directories shouldn't matter - we don't
read them.
String pathToCheck = workersPath;
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java
b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java
index 454d503..7a4274e 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java
@@ -13,6 +13,8 @@
*/
package org.apache.hadoop.hive.registry.impl;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +34,7 @@ public class ZookeeperUtils {
public static String setupZookeeperAuth(Configuration conf, String
saslLoginContextName,
String zkPrincipal, String zkKeytab) throws IOException {
// If the login context name is not set, we are in the client and don't
need auth.
- if (UserGroupInformation.isSecurityEnabled() && saslLoginContextName !=
null) {
+ if (isKerberosEnabled(conf) && saslLoginContextName != null) {
LOG.info("UGI security is enabled. Setting up ZK auth.");
if (zkPrincipal == null || zkPrincipal.isEmpty()) {
@@ -53,7 +55,19 @@ public class ZookeeperUtils {
}
/**
- * Dynamically sets up the JAAS configuration that uses kerberos
+ * Check if Kerberos is enabled.
+ */
+ public static boolean isKerberosEnabled(Configuration conf) {
+ try {
+ return UserGroupInformation.getLoginUser().isFromKeytab() &&
!AuthenticationMethod.SIMPLE.name().equalsIgnoreCase(
+ HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION));
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Dynamically sets up the JAAS configuration that uses kerberos.
*
* @param principal
* @param keyTabFile
diff --git
a/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java
b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java
new file mode 100644
index 0000000..c486274
--- /dev/null
+++
b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.registry.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * ZookeeperUtils test suite.
+ */
+public class TestZookeeperUtils {
+
+ private Configuration conf;
+ private UserGroupInformation ugi;
+
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ ugi = Mockito.mock(UserGroupInformation.class);
+ UserGroupInformation.setLoginUser(ugi);
+ }
+
+ @Test
+ public void testHadoopKerberosZookeeperDefault() {
+ Mockito.when(ugi.isFromKeytab()).thenReturn(true);
+ Assert.assertTrue(ZookeeperUtils.isKerberosEnabled(conf));
+ }
+
+ @Test
+ public void testHadoopKerberosZookeeperSimple(){
+ Mockito.when(ugi.isFromKeytab()).thenReturn(true);
+ conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname,
+ AuthenticationMethod.SIMPLE.name());
+ Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf));
+ }
+
+ @Test
+ public void testHadoopSimpleZookeeperDefault(){
+ Mockito.when(ugi.isFromKeytab()).thenReturn(false);
+ conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname,
+ AuthenticationMethod.SIMPLE.name());
+ Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf));
+ }
+}
diff --git
a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java
b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java
index f87b610..122742e 100644
---
a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java
+++
b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java
@@ -40,7 +40,7 @@ public class HS2ActivePassiveHARegistryClient {
String namespace = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE);
Preconditions.checkArgument(!StringUtils.isBlank(namespace),
HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE.varname + "
cannot be null or empty");
- String nsKey = ZkRegistryBase.getRootNamespace(null, namespace + "-");
+ String nsKey = ZkRegistryBase.getRootNamespace(conf, null, namespace +
"-");
HS2ActivePassiveHARegistry registry = hs2Registries.get(nsKey);
if (registry == null) {
registry = HS2ActivePassiveHARegistry.create(conf, true);
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java
b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index 5d81668..17570f7 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -86,9 +86,9 @@ import
org.apache.hadoop.hive.ql.session.ClearDanglingScratchDir;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.txn.compactor.CompactorThread;
import org.apache.hadoop.hive.ql.txn.compactor.Worker;
+import org.apache.hadoop.hive.registry.impl.ZookeeperUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.hive.common.util.ShutdownHookManager;
@@ -278,6 +278,7 @@ public class HiveServer2 extends CompositeService {
}
wmQueue =
hiveConf.get(ConfVars.HIVE_SERVER2_TEZ_INTERACTIVE_QUEUE.varname, "").trim();
+ this.zooKeeperAclProvider = getACLProvider(hiveConf);
this.serviceDiscovery =
hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY);
this.activePassiveHA =
hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_ENABLE);
@@ -453,29 +454,34 @@ public class HiveServer2 extends CompositeService {
/**
* ACLProvider for providing appropriate ACLs to CuratorFrameworkFactory
*/
- private final ACLProvider zooKeeperAclProvider = new ACLProvider() {
+ private ACLProvider zooKeeperAclProvider;
- @Override
- public List<ACL> getDefaultAcl() {
- List<ACL> nodeAcls = new ArrayList<ACL>();
- if (UserGroupInformation.isSecurityEnabled()) {
- // Read all to the world
- nodeAcls.addAll(Ids.READ_ACL_UNSAFE);
- // Create/Delete/Write/Admin to the authenticated user
- nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS));
- } else {
- // ACLs for znodes on a non-kerberized cluster
- // Create/Read/Delete/Write/Admin to the world
- nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE);
+ private ACLProvider getACLProvider(HiveConf hiveConf) {
+ final boolean isSecure = ZookeeperUtils.isKerberosEnabled(hiveConf);
+
+ return new ACLProvider() {
+ @Override
+ public List<ACL> getDefaultAcl() {
+ List<ACL> nodeAcls = new ArrayList<ACL>();
+ if (isSecure) {
+ // Read all to the world
+ nodeAcls.addAll(Ids.READ_ACL_UNSAFE);
+ // Create/Delete/Write/Admin to the authenticated user
+ nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS));
+ } else {
+ // ACLs for znodes on a non-kerberized cluster
+ // Create/Read/Delete/Write/Admin to the world
+ nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE);
+ }
+ return nodeAcls;
}
- return nodeAcls;
- }
- @Override
- public List<ACL> getAclForPath(String path) {
- return getDefaultAcl();
- }
- };
+ @Override
+ public List<ACL> getAclForPath(String path) {
+ return getDefaultAcl();
+ }
+ };
+ }
/**
* Add conf keys, values that HiveServer2 will publish to ZooKeeper.
@@ -522,7 +528,7 @@ public class HiveServer2 extends CompositeService {
* @throws Exception
*/
private void setUpZooKeeperAuth(HiveConf hiveConf) throws Exception {
- if (UserGroupInformation.isSecurityEnabled()) {
+ if (ZookeeperUtils.isKerberosEnabled(hiveConf)) {
String principal =
hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL);
if (principal.isEmpty()) {
throw new IOException("HiveServer2 Kerberos principal is empty");
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
index 953c5fd..ba1f177 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
@@ -34,6 +34,7 @@ import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
import
org.apache.hadoop.security.token.delegation.MetastoreDelegationTokenSupport;
import org.apache.zookeeper.CreateMode;
@@ -111,7 +112,8 @@ public class ZooKeeperTokenStore implements
DelegationTokenStore {
}
private void setupJAASConfig(Configuration conf) throws IOException {
- if (!UserGroupInformation.getLoginUser().isFromKeytab()) {
+ if (!UserGroupInformation.getLoginUser().isFromKeytab() ||
AuthenticationMethod.SIMPLE.name().equalsIgnoreCase(
+ getNonEmptyConfVar(conf, "hive.security.zookeeper.authentication"))) {
// The process has not logged in using keytab
// this should be a test mode, can't use keytab to authenticate
// with zookeeper.
