This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi-shaded.git
The following commit(s) were added to refs/heads/master by this push:
new dc1675b [KYUUBI-SHADED #40] kyuubi-relocated-hive-metastore-client
support ZooKeeper 3.4
dc1675b is described below
commit dc1675b3f69a77614b0706fc1c8dcb2eef9ff2ac
Author: zhouyifan279 <[email protected]>
AuthorDate: Thu Feb 29 17:53:17 2024 +0800
[KYUUBI-SHADED #40] kyuubi-relocated-hive-metastore-client support
ZooKeeper 3.4
### _Why are the changes needed?_
kyuubi-relocated-hive-metastore-client dependes on classed added in
ZooKeeper 3.6 to set SSL configs for ZK client.
As SSL configs can be set through JVM System properties and we need to
support ZooKeeper 3.4 client, these codes can be removed.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #40 from zhouyifan279/support-zk-34.
fe0b060 [zhouyifan279] Support ZooKeeper 3.4
Authored-by: zhouyifan279 <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
kyuubi-relocated-hive-metastore-client/pom.xml | 3 +-
.../shaded/hive/common/SSLZookeeperFactory.java | 84 ----------------------
.../shaded/hive/common/ZooKeeperHiveHelper.java | 71 ++----------------
.../shaded/hive/metastore/conf/MetastoreConf.java | 59 +--------------
4 files changed, 9 insertions(+), 208 deletions(-)
diff --git a/kyuubi-relocated-hive-metastore-client/pom.xml
b/kyuubi-relocated-hive-metastore-client/pom.xml
index 8951bc2..84d7adb 100644
--- a/kyuubi-relocated-hive-metastore-client/pom.xml
+++ b/kyuubi-relocated-hive-metastore-client/pom.xml
@@ -48,8 +48,7 @@ under the License.
<dependencies>
<dependency>
<groupId>org.apache.kyuubi</groupId>
- <!-- TODO: supports kyuubi-relocated-zookeeper-34 -->
- <artifactId>kyuubi-relocated-zookeeper-36</artifactId>
+ <artifactId>kyuubi-relocated-zookeeper-34</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
diff --git
a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/SSLZookeeperFactory.java
b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/SSLZookeeperFactory.java
deleted file mode 100644
index 9437859..0000000
---
a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/SSLZookeeperFactory.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.shaded.hive.common;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kyuubi.shaded.curator.utils.ZookeeperFactory;
-import org.apache.kyuubi.shaded.zookeeper.Watcher;
-import org.apache.kyuubi.shaded.zookeeper.ZooKeeper;
-import org.apache.kyuubi.shaded.zookeeper.client.ZKClientConfig;
-import org.apache.kyuubi.shaded.zookeeper.common.ClientX509Util;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Factory to create Zookeeper clients with the zookeeper.client.secure
enabled, allowing SSL
- * communication with the Zookeeper server.
- */
-public class SSLZookeeperFactory implements ZookeeperFactory {
-
- private static final Logger LOG =
LoggerFactory.getLogger(SSLZookeeperFactory.class);
-
- private boolean sslEnabled;
- private String keyStoreLocation;
- private String keyStorePassword;
- private String trustStoreLocation;
- private String trustStorePassword;
-
- public SSLZookeeperFactory(
- boolean sslEnabled,
- String keyStoreLocation,
- String keyStorePassword,
- String trustStoreLocation,
- String trustStorePassword) {
-
- this.sslEnabled = sslEnabled;
- this.keyStoreLocation = keyStoreLocation;
- this.keyStorePassword = keyStorePassword;
- this.trustStoreLocation = trustStoreLocation;
- this.trustStorePassword = trustStorePassword;
- if (sslEnabled) {
- if (StringUtils.isEmpty(keyStoreLocation)) {
- LOG.warn("Missing keystoreLocation parameter");
- }
- if (StringUtils.isEmpty(trustStoreLocation)) {
- LOG.warn("Missing trustStoreLocation parameter");
- }
- }
- }
-
- @Override
- public ZooKeeper newZooKeeper(
- String connectString, int sessionTimeout, Watcher watcher, boolean
canBeReadOnly)
- throws Exception {
- if (!this.sslEnabled) {
- return new ZooKeeper(connectString, sessionTimeout, watcher,
canBeReadOnly);
- }
- ZKClientConfig clientConfig = new ZKClientConfig();
- clientConfig.setProperty(ZKClientConfig.SECURE_CLIENT, "true");
- clientConfig.setProperty(
- ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET,
"org.apache.zookeeper.ClientCnxnSocketNetty");
- ClientX509Util x509Util = new ClientX509Util();
- clientConfig.setProperty(x509Util.getSslKeystoreLocationProperty(),
this.keyStoreLocation);
- clientConfig.setProperty(x509Util.getSslKeystorePasswdProperty(),
this.keyStorePassword);
- clientConfig.setProperty(x509Util.getSslTruststoreLocationProperty(),
this.trustStoreLocation);
- clientConfig.setProperty(x509Util.getSslTruststorePasswdProperty(),
this.trustStorePassword);
- return new ZooKeeper(connectString, sessionTimeout, watcher,
canBeReadOnly, clientConfig);
- }
-}
diff --git
a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZooKeeperHiveHelper.java
b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZooKeeperHiveHelper.java
index fda077b..1829624 100644
---
a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZooKeeperHiveHelper.java
+++
b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZooKeeperHiveHelper.java
@@ -29,6 +29,8 @@ import
org.apache.kyuubi.shaded.curator.framework.api.ACLProvider;
import org.apache.kyuubi.shaded.curator.framework.recipes.nodes.PersistentNode;
import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry;
import org.apache.kyuubi.shaded.curator.retry.RetryOneTime;
+import org.apache.kyuubi.shaded.curator.utils.DefaultZookeeperFactory;
+import org.apache.kyuubi.shaded.curator.utils.ZookeeperFactory;
import org.apache.kyuubi.shaded.zookeeper.CreateMode;
import org.apache.kyuubi.shaded.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -65,11 +67,6 @@ public class ZooKeeperHiveHelper {
private int sessionTimeout;
private int baseSleepTime;
private int maxRetries;
- private boolean sslEnabled = false;
- private String keyStoreLocation = null;
- private String keyStorePassword = null;
- private String trustStoreLocation = null;
- private String trustStorePassword = null;
public ZooKeeperHiveHelper build() {
return new ZooKeeperHiveHelper(this);
@@ -110,31 +107,6 @@ public class ZooKeeperHiveHelper {
return this;
}
- public ZooKeeperHiveHelperBuilder sslEnabled(boolean sslEnabled) {
- this.sslEnabled = sslEnabled;
- return this;
- }
-
- public ZooKeeperHiveHelperBuilder keyStoreLocation(String
keyStoreLocation) {
- this.keyStoreLocation = keyStoreLocation;
- return this;
- }
-
- public ZooKeeperHiveHelperBuilder keyStorePassword(String
keyStorePassword) {
- this.keyStorePassword = keyStorePassword;
- return this;
- }
-
- public ZooKeeperHiveHelperBuilder trustStoreLocation(String
trustStoreLocation) {
- this.trustStoreLocation = trustStoreLocation;
- return this;
- }
-
- public ZooKeeperHiveHelperBuilder trustStorePassword(String
trustStorePassword) {
- this.trustStorePassword = trustStorePassword;
- return this;
- }
-
public String getQuorum() {
return quorum;
}
@@ -162,26 +134,6 @@ public class ZooKeeperHiveHelper {
public int getMaxRetries() {
return maxRetries;
}
-
- public boolean isSslEnabled() {
- return sslEnabled;
- }
-
- public String getKeyStoreLocation() {
- return keyStoreLocation;
- }
-
- public String getKeyStorePassword() {
- return keyStorePassword;
- }
-
- public String getTrustStoreLocation() {
- return trustStoreLocation;
- }
-
- public String getTrustStorePassword() {
- return trustStorePassword;
- }
}
public static ZooKeeperHiveHelper.ZooKeeperHiveHelperBuilder builder() {
@@ -194,9 +146,8 @@ public class ZooKeeperHiveHelper {
private int sessionTimeout;
private int baseSleepTime;
private int maxRetries;
- private boolean sslEnabled;
- private SSLZookeeperFactory sslZookeeperFactory;
+ private ZookeeperFactory zookeeperFactory;
private CuratorFramework zooKeeperClient;
private boolean deregisteredWithZooKeeper = false; // Set to true only when
deregistration happens
private PersistentNode znode;
@@ -224,14 +175,7 @@ public class ZooKeeperHiveHelper {
this.sessionTimeout = builder.getSessionTimeout();
this.baseSleepTime = builder.getBaseSleepTime();
this.maxRetries = builder.getMaxRetries();
- this.sslEnabled = builder.isSslEnabled();
- this.sslZookeeperFactory =
- new SSLZookeeperFactory(
- sslEnabled,
- builder.getKeyStoreLocation(),
- builder.getKeyStorePassword(),
- builder.getTrustStoreLocation(),
- builder.getTrustStorePassword());
+ this.zookeeperFactory = new DefaultZookeeperFactory();
}
/** Get the ensemble server addresses. The format is: host1:port,
host2:port.. */
@@ -339,21 +283,20 @@ public class ZooKeeperHiveHelper {
ACLProvider zooKeeperAclProvider, String nameSpace) {
LOG.info(
"Creating curator client with connectString: {} namespace: {}
sessionTimeoutMs: {}"
- + " connectionTimeoutMs: {} exponentialBackoff - sleepTime: {}
maxRetries: {} sslEnabled: {}",
+ + " connectionTimeoutMs: {} exponentialBackoff - sleepTime: {}
maxRetries: {}",
quorum,
nameSpace,
sessionTimeout,
connectionTimeout,
baseSleepTime,
- maxRetries,
- sslEnabled);
+ maxRetries);
// Create a CuratorFramework instance to be used as the ZooKeeper client.
// Use the zooKeeperAclProvider, when specified, to create appropriate
ACLs.
CuratorFrameworkFactory.Builder builder =
CuratorFrameworkFactory.builder()
.connectString(quorum)
.namespace(nameSpace)
- .zookeeperFactory(this.sslZookeeperFactory);
+ .zookeeperFactory(this.zookeeperFactory);
if (connectionTimeout > 0) {
builder = builder.connectionTimeoutMs(connectionTimeout);
}
diff --git
a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/MetastoreConf.java
b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/MetastoreConf.java
index c803ec4..358f125 100644
---
a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/MetastoreConf.java
+++
b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/MetastoreConf.java
@@ -145,11 +145,7 @@ public class MetastoreConf {
ConfVars.SSL_KEYSTORE_PASSWORD.varname,
ConfVars.SSL_KEYSTORE_PASSWORD.hiveName,
ConfVars.SSL_TRUSTSTORE_PASSWORD.varname,
- ConfVars.SSL_TRUSTSTORE_PASSWORD.hiveName,
- ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.varname,
- ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.hiveName,
- ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.varname,
- ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.hiveName);
+ ConfVars.SSL_TRUSTSTORE_PASSWORD.hiveName);
public static ConfVars getMetaConf(String name) {
return metaConfs.get(name);
@@ -308,40 +304,6 @@ public class MetastoreConf {
new TimeValidator(TimeUnit.MILLISECONDS),
"Initial amount of time (in milliseconds) to wait between retries\n"
+ "when connecting to the ZooKeeper server when using
ExponentialBackoffRetry policy."),
- THRIFT_ZOOKEEPER_SSL_ENABLE(
- "metastore.zookeeper.ssl.client.enable",
- "hive.zookeeper.ssl.client.enable",
- false,
- "Set client to use TLS when connecting to ZooKeeper. An explicit
value overrides any value set via the "
- + "zookeeper.client.secure system property (note the different
name). Defaults to false if neither is set."),
- THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION(
- "metastore.zookeeper.ssl.keystore.location",
- "hive.zookeeper.ssl.keystore.location",
- "",
- "Keystore location when using a client-side certificate with TLS
connectivity to ZooKeeper. "
- + "Overrides any explicit value set via the
zookeeper.ssl.keyStore.location "
- + "system property (note the camelCase)."),
- THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD(
- "metastore.zookeeper.ssl.keystore.password",
- "hive.zookeeper.ssl.keystore.password",
- "",
- "Keystore password when using a client-side certificate with TLS
connectivity to ZooKeeper."
- + "Overrides any explicit value set via the
zookeeper.ssl.keyStore.password"
- + "system property (note the camelCase)."),
- THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION(
- "metastore.zookeeper.ssl.truststore.location",
- "hive.zookeeper.ssl.truststore.location",
- "",
- "Truststore location when using a client-side certificate with TLS
connectivity to ZooKeeper. "
- + "Overrides any explicit value set via the
zookeeper.ssl.trustStore.location "
- + "system property (note the camelCase)."),
- THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD(
- "metastore.zookeeper.ssl.truststore.password",
- "hive.zookeeper.ssl.truststore.password",
- "",
- "Truststore password when using a client-side certificate with TLS
connectivity to ZooKeeper."
- + "Overrides any explicit value set via the
zookeeper.ssl.trustStore.password "
- + "system property (note the camelCase)."),
THRIFT_URI_SELECTION(
"metastore.thrift.uri.selection",
"hive.metastore.uri.selection",
@@ -1176,18 +1138,6 @@ public class MetastoreConf {
}
public static ZooKeeperHiveHelper getZKConfig(Configuration conf) {
- String keyStorePassword = "";
- String trustStorePassword = "";
- if (MetastoreConf.getBoolVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_ENABLE)) {
- try {
- keyStorePassword =
- MetastoreConf.getPassword(conf,
ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD);
- trustStorePassword =
- MetastoreConf.getPassword(conf,
ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD);
- } catch (IOException e) {
- throw new RuntimeException("Failed to read zookeeper configuration
passwords", e);
- }
- }
return ZooKeeperHiveHelper.builder()
.quorum(MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS))
.clientPort(MetastoreConf.getVar(conf,
ConfVars.THRIFT_ZOOKEEPER_CLIENT_PORT))
@@ -1207,13 +1157,6 @@ public class MetastoreConf {
ConfVars.THRIFT_ZOOKEEPER_CONNECTION_BASESLEEPTIME,
TimeUnit.MILLISECONDS))
.maxRetries(MetastoreConf.getIntVar(conf,
ConfVars.THRIFT_ZOOKEEPER_CONNECTION_MAX_RETRIES))
- .sslEnabled(MetastoreConf.getBoolVar(conf,
ConfVars.THRIFT_ZOOKEEPER_SSL_ENABLE))
- .keyStoreLocation(
- MetastoreConf.getVar(conf,
ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION))
- .keyStorePassword(keyStorePassword)
- .trustStoreLocation(
- MetastoreConf.getVar(conf,
ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION))
- .trustStorePassword(trustStorePassword)
.build();
}
