[
https://issues.apache.org/jira/browse/HIVE-21456?focusedWorklogId=758775&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-758775
]
ASF GitHub Bot logged work on HIVE-21456:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 19/Apr/22 19:32
Start Date: 19/Apr/22 19:32
Worklog Time Spent: 10m
Work Description: sourabh912 commented on code in PR #3105:
URL: https://github.com/apache/hive/pull/3105#discussion_r853417994
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java:
##########
@@ -343,21 +366,162 @@ public static void startMetaStore(int port,
HadoopThriftAuthBridge bridge,
startMetaStore(port, bridge, conf, false, null);
}
- /**
- * Start Metastore based on a passed {@link HadoopThriftAuthBridge}.
- *
- * @param port The port on which the Thrift server will start to serve
- * @param bridge
- * @param conf Configuration overrides
- * @param startMetaStoreThreads Start the background threads (initiator,
cleaner, statsupdater, etc.)
- * @param startedBackgroundThreads If startMetaStoreThreads is true, this
AtomicBoolean will be switched to true,
- * when all of the background threads are scheduled. Useful for testing
purposes to wait
- * until the MetaStore is fully initialized.
- * @throws Throwable
- */
- public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
- Configuration conf, boolean startMetaStoreThreads, AtomicBoolean
startedBackgroundThreads) throws Throwable {
- isMetaStoreRemote = true;
+ public static boolean isThriftServerRunning() {
+ return thriftServer != null && thriftServer.isRunning();
+ }
+
+ // TODO: Is it worth trying to use a server that supports HTTP/2?
+ // Does the Thrift http client support this?
+
+ public static ThriftServer startHttpMetastore(int port, Configuration conf)
+ throws Exception {
+ LOG.info("Attempting to start http metastore server on port: {}", port);
+
+ // This check is likely pointless, especially with the current state of
the http
+ // servlet which respects whatever comes in. Putting this in place for the
moment
+ // only to enable testing on an otherwise secure cluster.
+ LOG.info(" Checking if security is enabled");
+ if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.info("Logging in via keytab while starting HTTP metastore");
+ // Handle renewal
+ String kerberosName =
SecurityUtil.getServerPrincipal(MetastoreConf.getVar(conf,
ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0");
+ String keyTabFile = MetastoreConf.getVar(conf,
ConfVars.KERBEROS_KEYTAB_FILE);
+ UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile);
+ } else {
+ LOG.info("Security is not enabled. Not logging in via keytab");
+ }
+
+ // TODO Bunch of http specific variables need to be defined here.
+ long maxMessageSize = MetastoreConf.getLongVar(conf,
ConfVars.SERVER_MAX_MESSAGE_SIZE);
+ int minWorkerThreads = MetastoreConf.getIntVar(conf,
ConfVars.SERVER_MIN_THREADS);
+ int maxWorkerThreads = MetastoreConf.getIntVar(conf,
ConfVars.SERVER_MAX_THREADS);
+
+ boolean useCompactProtocol = MetastoreConf.getBoolVar(conf,
ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
+
+ // Server thread pool
+ // Start with minWorkerThreads, expand till maxWorkerThreads and reject
+ // subsequent requests
+ String threadPoolName = "HiveServer2-HttpHandler-Pool";
+ ExecutorService executorService = new ThreadPoolExecutor(
+ minWorkerThreads, maxWorkerThreads, 60, TimeUnit.SECONDS,
+ new SynchronousQueue<>());
+
+ ExecutorThreadPool threadPool = new
ExecutorThreadPool((ThreadPoolExecutor) executorService);
+
+ // HTTP Server
+ org.eclipse.jetty.server.Server server = new Server(threadPool);
+ server.setStopAtShutdown(true);
+
+ ServerConnector connector;
+
+ final HttpConfiguration httpServerConf = new HttpConfiguration();
+ // TODO: Read from Configuration
+ httpServerConf.setRequestHeaderSize(
+ MetastoreConf.getIntVar(conf,
ConfVars.METASTORE_THRIFT_HTTP_REQUEST_HEADER_SIZE));
+ httpServerConf.setResponseHeaderSize(
+ MetastoreConf.getIntVar(conf,
ConfVars.METASTORE_THRIFT_HTTP_RESPONSE_HEADER_SIZE));
+
+ final HttpConnectionFactory http = new
HttpConnectionFactory(httpServerConf);
+
+ boolean useSsl = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
+ String schemeName = useSsl ? "https" : "http";
+ if (useSsl) {
+ String keyStorePath = MetastoreConf.getVar(conf,
ConfVars.SSL_KEYSTORE_PATH).trim();
+ if (keyStorePath.isEmpty()) {
+ throw new
IllegalArgumentException(ConfVars.SSL_KEYSTORE_PATH.toString()
+ + " Not configured for SSL connection");
+ }
+ String keyStorePassword =
+ MetastoreConf.getPassword(conf,
MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD);
+ String keyStoreType =
+ MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_TYPE).trim();
+ String keyStoreAlgorithm =
+ MetastoreConf.getVar(conf,
ConfVars.SSL_KEYMANAGERFACTORY_ALGORITHM).trim();
+
+ SslContextFactory sslContextFactory = new SslContextFactory();
+ // TODO: Add support for excluding protocols?
+ String[] excludedProtocols = MetastoreConf.getVar(conf,
ConfVars.SSL_PROTOCOL_BLACKLIST).split(",");
+ LOG.info("HTTP Server SSL: adding excluded protocols: " +
Arrays.toString(excludedProtocols));
+ sslContextFactory.addExcludeProtocols(excludedProtocols);
+ LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = "
+ + Arrays.toString(sslContextFactory.getExcludeProtocols()));
+ sslContextFactory.setKeyStorePath(keyStorePath);
+ sslContextFactory.setKeyStorePassword(keyStorePassword);
+ sslContextFactory.setKeyStoreType(keyStoreType);
+ sslContextFactory.setKeyManagerFactoryAlgorithm(keyStoreAlgorithm);
+ connector = new ServerConnector(server, sslContextFactory, http);
+ } else {
+ connector = new ServerConnector(server, http);
+ }
+ connector.setPort(port);
+ connector.setReuseAddress(true);
+ // TODO: What should the idle timeout be for the metastore. 30 minutes
seems a little too long.
+ connector.setIdleTimeout(120 * 1000);
+ // TODO: AcceptQueueSize needs to be higher for HMS
+ connector.setAcceptQueueSize(maxWorkerThreads);
+ // TODO: Connection keepalive configuration?
+
+ server.addConnector(connector);
+
+ TProcessor processor;
+
+ // All of this code can be re-used.
+ // Eventually move the HTTP and Binary parts into separate
+ // classes.
+ final TProtocolFactory protocolFactory;
+ final TProtocolFactory inputProtoFactory;
+ if (useCompactProtocol) {
+ protocolFactory = new TCompactProtocol.Factory();
+ inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize,
maxMessageSize);
+ } else {
+ protocolFactory = new TBinaryProtocol.Factory();
+ inputProtoFactory = new TBinaryProtocol.Factory(true, true,
maxMessageSize, maxMessageSize);
+ }
+
+ // TODO ZZZ: HMS seems to have it's own set of handlers. Not sure if the
threadpool here is actually required.
+ HMSHandler baseHandler = new HMSHandler("new db based metaserver",
+ conf);
+ IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
+ processor = new ThriftHiveMetastore.Processor<>(handler);
+ LOG.info("Starting DB backed MetaStore Server with generic processor");
Review Comment:
Sure.
Issue Time Tracking
-------------------
Worklog Id: (was: 758775)
Time Spent: 6.5h (was: 6h 20m)
> Hive Metastore Thrift over HTTP
> -------------------------------
>
> Key: HIVE-21456
> URL: https://issues.apache.org/jira/browse/HIVE-21456
> Project: Hive
> Issue Type: New Feature
> Components: Metastore, Standalone Metastore
> Reporter: Amit Khanna
> Assignee: Sourabh Goyal
> Priority: Major
> Labels: pull-request-available
> Attachments: HIVE-21456.2.patch, HIVE-21456.3.patch,
> HIVE-21456.4.patch, HIVE-21456.patch
>
> Time Spent: 6.5h
> Remaining Estimate: 0h
>
> Hive Metastore currently doesn't have support for HTTP transport because of
> which it is not possible to access it via Knox. Adding support for Thrift
> over HTTP transport will allow the clients to access via Knox
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
