[hive] branch master updated: HIVE-27129: Add enhanced support for Hive Metastore Client http support (Junlin Zeng, reviewed by Vihang Karajgaonkar)
This is an automated email from the ASF dual-hosted git repository.
vihangk1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 63a02465957 HIVE-27129: Add enhanced support for Hive Metastore Client
http support (Junlin Zeng, reviewed by Vihang Karajgaonkar)
63a02465957 is described below
commit 63a02465957abe5311d287e79cd9cb949170dedf
Author: Junlin Zeng <69210845+junlinzeng...@users.noreply.github.com>
AuthorDate: Fri Mar 10 13:14:43 2023 -0800
HIVE-27129: Add enhanced support for Hive Metastore Client http support
(Junlin Zeng, reviewed by Vihang Karajgaonkar)
(Closes #4104)
---
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 95 --
.../hadoop/hive/metastore/conf/MetastoreConf.java | 3 +
.../metastore/TestHiveMetastoreHttpHeaders.java| 108 +
3 files changed, 176 insertions(+), 30 deletions(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 50412dd17be..e229ab1a0b4 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -46,6 +46,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
+import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.TimeUnit;
@@ -54,6 +55,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.security.auth.login.LoginException;
import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -621,6 +623,24 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
return transport;
}
+ private Map getAdditionalHeaders() {
+Map headers = new HashMap<>();
+String keyValuePairs = MetastoreConf.getVar(conf,
+ConfVars.METASTORE_CLIENT_ADDITIONAL_HEADERS);
+try {
+ List headerKeyValues =
+ Splitter.on(',').trimResults().splitToList(keyValuePairs);
+ for (String header : headerKeyValues) {
+String[] parts = header.split("=");
+headers.put(parts[0].trim(), parts[1].trim());
+ }
+} catch (Exception ex) {
+ LOG.warn("Could not parse the headers provided in "
+ + ConfVars.METASTORE_CLIENT_ADDITIONAL_HEADERS, ex);
+}
+return headers;
+ }
+
/*
Creates a THttpClient if HTTP mode is enabled. If Client auth mode is set to
JWT,
then the method fetches JWT from environment variable: HMS_JWT and sets in
auth
@@ -629,10 +649,47 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
private THttpClient createHttpClient(URI store, boolean useSSL) throws
MetaException,
TTransportException {
String path = MetaStoreUtils.getHttpPath(MetastoreConf.getVar(conf,
ConfVars.THRIFT_HTTP_PATH));
-String httpUrl = (useSSL ? "https://"; : "http://";) + store.getHost() + ":"
+ store.getPort() + path;
+String urlScheme;
+if (useSSL || Objects.equals(store.getScheme(), "https")) {
+ urlScheme = "https://";;
+} else {
+ urlScheme = "http://";;
+}
+String httpUrl = urlScheme + store.getHost() + ":" + store.getPort() +
path;
+
+HttpClientBuilder httpClientBuilder = createHttpClientBuilder();
+THttpClient tHttpClient;
+try {
+ if (useSSL) {
+String trustStorePath = MetastoreConf.getVar(conf,
ConfVars.SSL_TRUSTSTORE_PATH).trim();
+if (trustStorePath.isEmpty()) {
+ throw new IllegalArgumentException(ConfVars.SSL_TRUSTSTORE_PATH + "
Not configured for SSL connection");
+}
+String trustStorePassword = MetastoreConf.getPassword(conf,
MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
+String trustStoreType = MetastoreConf.getVar(conf,
ConfVars.SSL_TRUSTSTORE_TYPE).trim();
+String trustStoreAlgorithm = MetastoreConf.getVar(conf,
ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
+tHttpClient =
+SecurityUtils.getThriftHttpsClient(httpUrl, trustStorePath,
trustStorePassword, trustStoreAlgorithm,
+trustStoreType, httpClientBuilder);
+ } else {
+tHttpClient = new THttpClient(httpUrl, httpClientBuilder.build());
+ }
+} catch (Exception e) {
+ if (e instanceof TTransportException) {
+throw (TTransportException) e;
+ } else {
+throw new MetaException("Failed to create http transport client to
url: " + httpUrl + ". Error:" + e);
+ }
[hive] branch master updated: HIVE-27116: HS2 need to send owner info for UDFs in the HivePrivilegeObject for authorization (Sai Hemanth Gantasala, reviewed by Zhihua Deng)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 62b1c7bf3b9 HIVE-27116: HS2 need to send owner info for UDFs in the
HivePrivilegeObject for authorization (Sai Hemanth Gantasala, reviewed by
Zhihua Deng)
62b1c7bf3b9 is described below
commit 62b1c7bf3b91a296e466774a6cb3ec31151e4067
Author: Sai Hemanth Gantasala
<68923650+saihemanth-cloud...@users.noreply.github.com>
AuthorDate: Fri Mar 10 14:56:49 2023 +0530
HIVE-27116: HS2 need to send owner info for UDFs in the HivePrivilegeObject
for authorization (Sai Hemanth Gantasala, reviewed by Zhihua Deng)
Closes #4092
---
.../hadoop/hive/ql/TestCreateUdfEntities.java | 4 +--
.../ql/ddl/function/AbstractFunctionAnalyzer.java | 24 +--
.../org/apache/hadoop/hive/ql/hooks/Entity.java| 36 +++---
.../apache/hadoop/hive/ql/hooks/ReadEntity.java| 1 +
.../apache/hadoop/hive/ql/hooks/WriteEntity.java | 6
.../authorization/command/CommandAuthorizerV2.java | 13 ++--
.../llap/authorization_functions_in_views.q.out| 4 +--
7 files changed, 74 insertions(+), 14 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java
index 325831e2eb9..0d7503668bc 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestCreateUdfEntities.java
@@ -59,7 +59,7 @@ public class TestCreateUdfEntities {
assertEquals("default", outputEntities[0].getDatabase().getName());
assertEquals(Entity.Type.FUNCTION, outputEntities[1].getType());
-assertEquals(funcName, outputEntities[1].getFunctionName());
+assertEquals(funcName, outputEntities[1].getFunction().getFunctionName());
assertEquals(Entity.Type.LOCAL_DIR, outputEntities[2].getType());
assertEquals("file:///tmp/udf1.jar",
outputEntities[2].getLocation().toString());
@@ -77,7 +77,7 @@ public class TestCreateUdfEntities {
assertEquals("default", outputEntities[0].getDatabase().getName());
assertEquals(Entity.Type.FUNCTION, outputEntities[1].getType());
-assertEquals(funcName, outputEntities[1].getFunctionName());
+assertEquals(funcName, outputEntities[1].getFunction().getFunctionName());
assertEquals(Entity.Type.DFS_DIR, outputEntities[2].getType());
assertEquals("hdfs:///tmp/udf1.jar",
outputEntities[2].getLocation().toString());
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java
index 997cb97d950..2ca87cca193 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java
@@ -21,6 +21,9 @@ package org.apache.hadoop.hive.ql.ddl.function;
import java.util.List;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.FunctionType;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.FunctionUtils;
@@ -29,6 +32,8 @@ import org.apache.hadoop.hive.ql.hooks.Entity.Type;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.session.SessionState;
/**
* Abstract ancestor of function related ddl analyzer classes.
@@ -66,11 +71,24 @@ public abstract class AbstractFunctionAnalyzer extends
BaseSemanticAnalyzer {
}
if (database != null) {
outputs.add(new WriteEntity(database,
WriteEntity.WriteType.DDL_NO_LOCK));
+ // Add the permanent function as a WriteEntity
+ Function function;
+ if (queryState.getHiveOperation().equals(HiveOperation.CREATEFUNCTION)) {
+function = new Function(functionName, database.getName(), className,
+SessionState.getUserFromAuthenticator(), PrincipalType.USER,
+(int) (System.currentTimeMillis() / 1000), FunctionType.JAVA,
resources);
+ } else {
+try {
+ function = db.getFunction(database.getName(), functionName);
+} catch (HiveException e) {
+ throw new RuntimeException(e);
+}
+ }
+ outputs.add(new WriteEntity(function,
WriteEntity.WriteType.DDL_NO_LOCK));
+} else { // Temporary functions
+ outputs.add(new
[hive] branch master updated: HIVE-27074: Make IMetaStoreClient extends AutoCloseable (#4054) (Hussein Awala, reviewed by Zhihua Deng)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6ad8583190c HIVE-27074: Make IMetaStoreClient extends AutoCloseable
(#4054) (Hussein Awala, reviewed by Zhihua Deng)
6ad8583190c is described below
commit 6ad8583190c153863e5bd237dd2dbd77c0e5203f
Author: Hussein Awala
AuthorDate: Fri Mar 10 10:16:22 2023 +0100
HIVE-27074: Make IMetaStoreClient extends AutoCloseable (#4054) (Hussein
Awala, reviewed by Zhihua Deng)
---
.../main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 44bd7e37dfa..9708566884a 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -43,7 +43,7 @@ import org.apache.thrift.TException;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public interface IMetaStoreClient {
+public interface IMetaStoreClient extends AutoCloseable {
/**
* Returns whether current client is compatible with conf argument or not
