Kevin Sweeney created AURORA-746:
------------------------------------
Summary: LocalSchedulerMain shutdown deadlock
Key: AURORA-746
URL: https://issues.apache.org/jira/browse/AURORA-746
Project: Aurora
Issue Type: Story
Components: Scheduler, Testing
Reporter: Kevin Sweeney
Looks like LocalSchedulerMain waits forever on an attempt to cancel its
zookeeper leadership
{noformat}
"qtp516227519-124" prio=10 tid=0x00007f65a8e16000 nid=0x4c50 waiting for
monitor entry [0x00007f648faf5000]
java.lang.Thread.State: BLOCKED (on object monitor)
at
com.twitter.common.zookeeper.Group$ActiveMembership.cancel(Group.java:365)
- waiting to lock <0x000000060f86d9a8> (a
com.twitter.common.zookeeper.Group$ActiveMembership)
at
com.twitter.common.zookeeper.ServerSetImpl$MemberStatus.leave(ServerSetImpl.java:255)
- locked <0x000000060f86ced0> (a
com.twitter.common.zookeeper.ServerSetImpl$MemberStatus)
at
com.twitter.common.zookeeper.ServerSetImpl$2.leave(ServerSetImpl.java:199)
at
com.twitter.common.zookeeper.SingletonService$1$1.leave(SingletonService.java:185)
- locked <0x000000060f86ce98> (a
com.twitter.common.zookeeper.SingletonService$1$1)
at
org.apache.aurora.scheduler.SchedulerLifecycle$8.execute(SchedulerLifecycle.java:367)
at
org.apache.aurora.scheduler.SchedulerLifecycle$8.execute(SchedulerLifecycle.java:352)
at com.twitter.common.base.Closures$4.execute(Closures.java:120)
at com.twitter.common.base.Closures$3.execute(Closures.java:98)
at
com.twitter.common.util.StateMachine.transition(StateMachine.java:191)
at
org.apache.aurora.scheduler.SchedulerLifecycle$4.execute(SchedulerLifecycle.java:232)
at
com.twitter.common.application.ShutdownRegistry$ShutdownRegistryImpl.execute(ShutdownRegistry.java:88)
- locked <0x000000060f86c6c0> (a
com.twitter.common.application.ShutdownRegistry$ShutdownRegistryImpl)
at com.twitter.common.application.Lifecycle.shutdown(Lifecycle.java:92)
- locked <0x000000060f86c6b0> (a java.lang.Object)
at
org.apache.aurora.scheduler.storage.backup.BackupModule$LifecycleHook.execute(BackupModule.java:117)
at
org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl$PendingRecovery$1.execute(Recovery.java:198)
at
org.apache.aurora.scheduler.storage.Storage$MutateWork$NoResult.apply(Storage.java:132)
at
org.apache.aurora.scheduler.storage.Storage$MutateWork$NoResult$Quiet.apply(Storage.java:149)
at
org.apache.aurora.scheduler.storage.mem.MemStorage.doWork(MemStorage.java:175)
at
org.apache.aurora.scheduler.storage.mem.MemStorage.access$400(MemStorage.java:59)
at
org.apache.aurora.scheduler.storage.mem.MemStorage$4.apply(MemStorage.java:200)
at
org.apache.aurora.scheduler.storage.mem.MemStorage$4.apply(MemStorage.java:197)
at
org.apache.aurora.scheduler.storage.db.DbStorage.write(DbStorage.java:150)
at
org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366.CGLIB$write$4(<generated>)
at
org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366$$FastClassByGuice$$9e51bc15.invoke(<generated>)
at
com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
org.mybatis.guice.transactional.TransactionalMethodInterceptor.invoke(TransactionalMethodInterceptor.java:101)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52)
at
org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366.write(<generated>)
at
org.apache.aurora.scheduler.storage.mem.MemStorage.write(MemStorage.java:197)
at
org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl$PendingRecovery.commit(Recovery.java:193)
at
org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl.commit(Recovery.java:182)
at
org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.commitRecovery(SchedulerThriftInterface.java:1009)
at
org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84.CGLIB$commitRecovery$7(<generated>)
at
org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84$$FastClassByGuice$$455b50c2.invoke(<generated>)
at
com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
org.apache.aurora.scheduler.thrift.aop.ServerInfoInterceptor.invoke(ServerInfoInterceptor.java:32)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
org.apache.aurora.scheduler.thrift.aop.ThriftStatsExporterInterceptor.invoke(ThriftStatsExporterInterceptor.java:47)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
org.apache.aurora.scheduler.thrift.aop.FeatureToggleInterceptor.invoke(FeatureToggleInterceptor.java:38)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
org.apache.aurora.scheduler.thrift.aop.UserCapabilityInterceptor.invoke(UserCapabilityInterceptor.java:92)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
org.apache.aurora.scheduler.thrift.aop.LoggingInterceptor.invoke(LoggingInterceptor.java:87)
at
com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
at
com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52)
at
org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84.commitRecovery(<generated>)
at
org.apache.aurora.gen.AuroraAdmin$Processor$commitRecovery.getResult(AuroraAdmin.java:1296)
at
org.apache.aurora.gen.AuroraAdmin$Processor$commitRecovery.getResult(AuroraAdmin.java:1281)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.thrift.server.TServlet.doPost(TServlet.java:83)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
at
com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263)
at
com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178)
at
com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:82)
at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:294)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
at
com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
at
com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118)
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1288)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:369)
at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
at
org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:944)
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1005)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
at
org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:667)
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
{noformat}
{noformat}
"main-EventThread" daemon prio=10 tid=0x00007f65a8e52000 nid=0x4c56 waiting on
condition [0x00007f648f4f3000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000076ed20ad8> (a
java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:236)
at
com.twitter.common.zookeeper.ZooKeeperClient.get(ZooKeeperClient.java:398)
- locked <0x000000060f86d060> (a
com.twitter.common.zookeeper.ZooKeeperClient)
at
com.twitter.common.zookeeper.ZooKeeperClient.get(ZooKeeperClient.java:332)
- locked <0x000000060f86d060> (a
com.twitter.common.zookeeper.ZooKeeperClient)
at
com.twitter.common.zookeeper.Group$ActiveMembership.join(Group.java:424)
- locked <0x000000060f86d9a8> (a
com.twitter.common.zookeeper.Group$ActiveMembership)
at
com.twitter.common.zookeeper.Group$ActiveMembership$4.get(Group.java:445)
at
com.twitter.common.zookeeper.Group$ActiveMembership$4.get(Group.java:442)
at com.twitter.common.util.BackoffHelper$1.get(BackoffHelper.java:109)
at com.twitter.common.util.BackoffHelper$1.get(BackoffHelper.java:107)
at
com.twitter.common.util.BackoffHelper.doUntilResult(BackoffHelper.java:127)
at
com.twitter.common.util.BackoffHelper.doUntilSuccess(BackoffHelper.java:107)
at
com.twitter.common.zookeeper.Group$ActiveMembership.tryJoin(Group.java:467)
- locked <0x000000060f86d9a8> (a
com.twitter.common.zookeeper.Group$ActiveMembership)
at
com.twitter.common.zookeeper.Group$ActiveMembership.access$800(Group.java:316)
at
com.twitter.common.zookeeper.Group$ActiveMembership$3.process(Group.java:433)
at
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:507)
{noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
