Rajesh Balamohan created HIVE-22751: ---------------------------------------
Summary: Move locking in HiveServer2::isDeregisteredWithZooKeeper to ZooKeeperHiveHelper Key: HIVE-22751 URL: https://issues.apache.org/jira/browse/HIVE-22751 Project: Hive Issue Type: Improvement Components: HiveServer2 Reporter: Rajesh Balamohan [https://github.com/apache/hive/blob/master/service/src/java/org/apache/hive/service/server/HiveServer2.java#L620] [https://github.com/apache/hive/blob/master/service/src/java/org/apache/hive/service/cli/session/SessionManager.java#L597] When queries are run in beeline and closed, it causes unwanted delays in shutting down beeline. Here is the threaddump from server side, which shows HiveServer2 lock contention. It would be good to move synchronization to "zooKeeperHelper.isDeregisteredWithZooKeeper" {noformat} "main" #1 prio=5 os_prio=0 tid=0x00007f78b0078800 nid=0x2d1c waiting on condition [0x00007f78b968c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000ac8d5ff0> (a java.util.concurrent.FutureTask) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429) at java.util.concurrent.FutureTask.get(FutureTask.java:191) at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:187) at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) - locked <0x00000000a9c5f2a8> (a java.lang.Object) at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:790) at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:763) at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:687) - locked <0x00000000a99bd568> (a org.apache.hive.service.server.HiveServer2) at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1016) at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:137) at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1294) at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1138) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.util.RunJar.run(RunJar.java:318) at org.apache.hadoop.util.RunJar.main(RunJar.java:232) "HiveServer2-HttpHandler-Pool: Thread-50" #50 prio=5 os_prio=0 tid=0x00007f78b3e60800 nid=0x2fa7 waiting for monitor entry [0x00007f7884edf000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hive.service.server.HiveServer2.isDeregisteredWithZooKeeper(HiveServer2.java:600) - waiting to lock <0x00000000a99bd568> (a org.apache.hive.service.server.HiveServer2) at org.apache.hive.service.cli.session.SessionManager.closeSessionInternal(SessionManager.java:631) at org.apache.hive.service.cli.session.SessionManager.closeSession(SessionManager.java:621) - locked <0x00000000aa1970b0> (a org.apache.hive.service.cli.session.SessionManager) at org.apache.hive.service.cli.CLIService.closeSession(CLIService.java:244) at org.apache.hive.service.cli.thrift.ThriftCLIService.CloseSession(ThriftCLIService.java:527) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$CloseSession.getResult(TCLIService.java:1517) at org.apache.hive.service.rpc.thrift.TCLIService$Processor$CloseSession.getResult(TCLIService.java:1502) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.thrift.server.TServlet.doPost(TServlet.java:83) at org.apache.hive.service.cli.thrift.ThriftHttpServlet.doPost(ThriftHttpServlet.java:237) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:848) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:513) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:539) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:259) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) {noformat} -- This message was sent by Atlassian Jira (v8.3.4#803005)