Repository: hive Updated Branches: refs/heads/master 4408661c0 -> ff2554328
HIVE-20237: Do Not Print StackTraces to STDERR in HiveMetaStore (Alice Fan, reviewed Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1f9c70e5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1f9c70e5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1f9c70e5 Branch: refs/heads/master Commit: 1f9c70e5148add278a1cb9b2186fb9a65d7ed1c5 Parents: 4408661 Author: Aihua Xu <aihu...@apache.org> Authored: Wed Aug 22 10:02:08 2018 -0700 Committer: Aihua Xu <aihu...@apache.org> Committed: Wed Aug 22 10:02:08 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/metastore/HiveMetaStore.java | 280 +++++++++---------- 1 file changed, 137 insertions(+), 143 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1f9c70e5/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 324035a..067eb5a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -9219,166 +9219,160 @@ public class HiveMetaStore extends ThriftHiveMetastore { public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, Configuration conf, Lock startLock, Condition startCondition, AtomicBoolean startedServing) throws Throwable { - try { - isMetaStoreRemote = true; - // Server will create new threads up to max as necessary. After an idle - // period, it will destroy threads to keep the number of threads in the - // pool to min. - long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE); - int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS); - int maxWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MAX_THREADS); - boolean tcpKeepAlive = MetastoreConf.getBoolVar(conf, ConfVars.TCP_KEEP_ALIVE); - boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT); - boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL); - boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL); - useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL); - - if (useSasl) { - // we are in secure mode. Login using keytab - String kerberosName = SecurityUtil - .getServerPrincipal(MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0"); - String keyTabFile = MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE); - UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile); - } - - TProcessor processor; - TTransportFactory transFactory; - final TProtocolFactory protocolFactory; - final TProtocolFactory inputProtoFactory; - if (useCompactProtocol) { - protocolFactory = new TCompactProtocol.Factory(); - inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize, maxMessageSize); + isMetaStoreRemote = true; + // Server will create new threads up to max as necessary. After an idle + // period, it will destroy threads to keep the number of threads in the + // pool to min. + long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE); + int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS); + int maxWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MAX_THREADS); + boolean tcpKeepAlive = MetastoreConf.getBoolVar(conf, ConfVars.TCP_KEEP_ALIVE); + boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT); + boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL); + boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL); + useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL); + + if (useSasl) { + // we are in secure mode. Login using keytab + String kerberosName = SecurityUtil + .getServerPrincipal(MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0"); + String keyTabFile = MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE); + UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile); + } + + TProcessor processor; + TTransportFactory transFactory; + final TProtocolFactory protocolFactory; + final TProtocolFactory inputProtoFactory; + if (useCompactProtocol) { + protocolFactory = new TCompactProtocol.Factory(); + inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize, maxMessageSize); + } else { + protocolFactory = new TBinaryProtocol.Factory(); + inputProtoFactory = new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize); + } + HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf, + false); + IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf); + + TServerSocket serverSocket; + + if (useSasl) { + // we are in secure mode. + if (useFramedTransport) { + throw new HiveMetaException("Framed transport is not supported with SASL enabled."); + } + saslServer = bridge.createServer( + MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE), + MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), + MetastoreConf.getVar(conf, ConfVars.CLIENT_KERBEROS_PRINCIPAL)); + // Start delegation token manager + delegationTokenManager = new MetastoreDelegationTokenManager(); + delegationTokenManager.startDelegationTokenSecretManager(conf, baseHandler, HadoopThriftAuthBridge.Server.ServerMode.METASTORE); + saslServer.setSecretManager(delegationTokenManager.getSecretManager()); + transFactory = saslServer.createTransportFactory( + MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL)); + processor = saslServer.wrapProcessor( + new ThriftHiveMetastore.Processor<>(handler)); + + LOG.info("Starting DB backed MetaStore Server in Secure Mode"); + } else { + // we are in unsecure mode. + if (MetastoreConf.getBoolVar(conf, ConfVars.EXECUTE_SET_UGI)) { + transFactory = useFramedTransport ? + new ChainedTTransportFactory(new TFramedTransport.Factory(), + new TUGIContainingTransport.Factory()) + : new TUGIContainingTransport.Factory(); + + processor = new TUGIBasedProcessor<>(handler); + LOG.info("Starting DB backed MetaStore Server with SetUGI enabled"); } else { - protocolFactory = new TBinaryProtocol.Factory(); - inputProtoFactory = new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize); + transFactory = useFramedTransport ? + new TFramedTransport.Factory() : new TTransportFactory(); + processor = new TSetIpAddressProcessor<>(handler); + LOG.info("Starting DB backed MetaStore Server"); } - HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf, - false); - IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf); - - TServerSocket serverSocket; + } - if (useSasl) { - // we are in secure mode. - if (useFramedTransport) { - throw new HiveMetaException("Framed transport is not supported with SASL enabled."); - } - saslServer = bridge.createServer( - MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE), - MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), - MetastoreConf.getVar(conf, ConfVars.CLIENT_KERBEROS_PRINCIPAL)); - // Start delegation token manager - delegationTokenManager = new MetastoreDelegationTokenManager(); - delegationTokenManager.startDelegationTokenSecretManager(conf, baseHandler, HadoopThriftAuthBridge.Server.ServerMode.METASTORE); - saslServer.setSecretManager(delegationTokenManager.getSecretManager()); - transFactory = saslServer.createTransportFactory( - MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL)); - processor = saslServer.wrapProcessor( - new ThriftHiveMetastore.Processor<>(handler)); - - LOG.info("Starting DB backed MetaStore Server in Secure Mode"); - } else { - // we are in unsecure mode. - if (MetastoreConf.getBoolVar(conf, ConfVars.EXECUTE_SET_UGI)) { - transFactory = useFramedTransport ? - new ChainedTTransportFactory(new TFramedTransport.Factory(), - new TUGIContainingTransport.Factory()) - : new TUGIContainingTransport.Factory(); - - processor = new TUGIBasedProcessor<>(handler); - LOG.info("Starting DB backed MetaStore Server with SetUGI enabled"); - } else { - transFactory = useFramedTransport ? - new TFramedTransport.Factory() : new TTransportFactory(); - processor = new TSetIpAddressProcessor<>(handler); - LOG.info("Starting DB backed MetaStore Server"); - } + if (!useSSL) { + serverSocket = SecurityUtils.getServerSocket(null, port); + } else { + String keyStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_PATH).trim(); + if (keyStorePath.isEmpty()) { + throw new IllegalArgumentException(ConfVars.SSL_KEYSTORE_PATH.toString() + + " Not configured for SSL connection"); } + String keyStorePassword = + MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD); - if (!useSSL) { - serverSocket = SecurityUtils.getServerSocket(null, port); - } else { - String keyStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_PATH).trim(); - if (keyStorePath.isEmpty()) { - throw new IllegalArgumentException(ConfVars.SSL_KEYSTORE_PATH.toString() - + " Not configured for SSL connection"); - } - String keyStorePassword = - MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD); - - // enable SSL support for HMS - List<String> sslVersionBlacklist = new ArrayList<>(); - for (String sslVersion : MetastoreConf.getVar(conf, ConfVars.SSL_PROTOCOL_BLACKLIST).split(",")) { - sslVersionBlacklist.add(sslVersion); - } - - serverSocket = SecurityUtils.getServerSSLSocket(null, port, keyStorePath, - keyStorePassword, sslVersionBlacklist); + // enable SSL support for HMS + List<String> sslVersionBlacklist = new ArrayList<>(); + for (String sslVersion : MetastoreConf.getVar(conf, ConfVars.SSL_PROTOCOL_BLACKLIST).split(",")) { + sslVersionBlacklist.add(sslVersion); } - if (tcpKeepAlive) { - serverSocket = new TServerSocketKeepAlive(serverSocket); - } + serverSocket = SecurityUtils.getServerSSLSocket(null, port, keyStorePath, + keyStorePassword, sslVersionBlacklist); + } - // Metrics will have already been initialized if we're using them since HMSHandler - // initializes them. - openConnections = Metrics.getOrCreateGauge(MetricsConstants.OPEN_CONNECTIONS); + if (tcpKeepAlive) { + serverSocket = new TServerSocketKeepAlive(serverSocket); + } - TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverSocket) - .processor(processor) - .transportFactory(transFactory) - .protocolFactory(protocolFactory) - .inputProtocolFactory(inputProtoFactory) - .minWorkerThreads(minWorkerThreads) - .maxWorkerThreads(maxWorkerThreads); + // Metrics will have already been initialized if we're using them since HMSHandler + // initializes them. + openConnections = Metrics.getOrCreateGauge(MetricsConstants.OPEN_CONNECTIONS); - TServer tServer = new TThreadPoolServer(args); - TServerEventHandler tServerEventHandler = new TServerEventHandler() { - @Override - public void preServe() { - } + TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverSocket) + .processor(processor) + .transportFactory(transFactory) + .protocolFactory(protocolFactory) + .inputProtocolFactory(inputProtoFactory) + .minWorkerThreads(minWorkerThreads) + .maxWorkerThreads(maxWorkerThreads); - @Override - public ServerContext createContext(TProtocol tProtocol, TProtocol tProtocol1) { - openConnections.incrementAndGet(); - return null; - } + TServer tServer = new TThreadPoolServer(args); + TServerEventHandler tServerEventHandler = new TServerEventHandler() { + @Override + public void preServe() { + } - @Override - public void deleteContext(ServerContext serverContext, TProtocol tProtocol, TProtocol tProtocol1) { - openConnections.decrementAndGet(); - // If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already - // cleaned up thread local RawStore. Otherwise, do it now. - cleanupRawStore(); - } + @Override + public ServerContext createContext(TProtocol tProtocol, TProtocol tProtocol1) { + openConnections.incrementAndGet(); + return null; + } - @Override - public void processContext(ServerContext serverContext, TTransport tTransport, TTransport tTransport1) { - } - }; + @Override + public void deleteContext(ServerContext serverContext, TProtocol tProtocol, TProtocol tProtocol1) { + openConnections.decrementAndGet(); + // If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already + // cleaned up thread local RawStore. Otherwise, do it now. + cleanupRawStore(); + } - tServer.setServerEventHandler(tServerEventHandler); - HMSHandler.LOG.info("Started the new metaserver on port [" + port - + "]..."); - HMSHandler.LOG.info("Options.minWorkerThreads = " - + minWorkerThreads); - HMSHandler.LOG.info("Options.maxWorkerThreads = " - + maxWorkerThreads); - HMSHandler.LOG.info("TCP keepalive = " + tcpKeepAlive); - HMSHandler.LOG.info("Enable SSL = " + useSSL); + @Override + public void processContext(ServerContext serverContext, TTransport tTransport, TTransport tTransport1) { + } + }; - boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL); - HMSHandler.LOG.info("Direct SQL optimization = {}", directSqlEnabled); + tServer.setServerEventHandler(tServerEventHandler); + HMSHandler.LOG.info("Started the new metaserver on port [" + port + + "]..."); + HMSHandler.LOG.info("Options.minWorkerThreads = " + + minWorkerThreads); + HMSHandler.LOG.info("Options.maxWorkerThreads = " + + maxWorkerThreads); + HMSHandler.LOG.info("TCP keepalive = " + tcpKeepAlive); + HMSHandler.LOG.info("Enable SSL = " + useSSL); - if (startLock != null) { - signalOtherThreadsToStart(tServer, startLock, startCondition, startedServing); - } - tServer.serve(); - } catch (Throwable x) { - x.printStackTrace(); - HMSHandler.LOG.error(StringUtils.stringifyException(x)); - throw x; + boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL); + HMSHandler.LOG.info("Direct SQL optimization = {}", directSqlEnabled); + + if (startLock != null) { + signalOtherThreadsToStart(tServer, startLock, startCondition, startedServing); } + tServer.serve(); } private static void cleanupRawStore() {
