[ https://issues.apache.org/jira/browse/GEODE-1894?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15490981#comment-15490981 ]
Dan Smith edited comment on GEODE-1894 at 9/15/16 12:14 AM: ------------------------------------------------------------ <edited - attached the thread dump as a file> was (Author: hitesh.khamesra): JNI global references: 6150 2016-09-14 09:33:59 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.66-b17 mixed mode): "RMI TCP Connection(idle)" #10169 daemon prio=5 os_prio=0 tid=0x00007f8f54003000 nid=0x2464 waiting on condition [0x00007f8fac9df000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0269958> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Attach Listener" #10071 daemon prio=9 os_prio=0 tid=0x00007f8fa8002000 nid=0x1cea waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5364 daemon prio=5 os_prio=0 tid=0x00007f8f4c826800 nid=0xa12 waiting on condition [0x00007f8ed08bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0ef4128> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AckReaderThread for : Event Processor for GatewaySender_ln.3" #5362 daemon prio=5 os_prio=0 tid=0x00007f8f18008800 nid=0xa11 runnable [0x00007f8ed0bbc000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at org.apache.geode.internal.cache.tier.sockets.Message.fetchHeader(Message.java:693) at org.apache.geode.internal.cache.tier.sockets.Message.readHeaderAndPayload(Message.java:710) at org.apache.geode.internal.cache.tier.sockets.Message.read(Message.java:661) at org.apache.geode.internal.cache.tier.sockets.Message.recv(Message.java:1103) - locked <0x00000000e0ffbdf8> (a java.nio.HeapByteBuffer) at org.apache.geode.cache.client.internal.GatewaySenderBatchOp$GatewaySenderGFEBatchOpImpl.attemptReadResponse(GatewaySenderBatchOp.java:205) at org.apache.geode.cache.client.internal.GatewaySenderBatchOp$GatewaySenderGFEBatchOpImpl.attemptRead(GatewaySenderBatchOp.java:166) at org.apache.geode.cache.client.internal.GatewaySenderBatchOp$GatewaySenderGFEBatchOpImpl.attempt(GatewaySenderBatchOp.java:145) at org.apache.geode.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:256) at org.apache.geode.cache.client.internal.pooling.PooledConnection.execute(PooledConnection.java:328) at org.apache.geode.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:937) at org.apache.geode.cache.client.internal.OpExecutorImpl.executeOn(OpExecutorImpl.java:579) at org.apache.geode.cache.client.internal.PoolImpl.executeOn(PoolImpl.java:748) at org.apache.geode.cache.client.internal.GatewaySenderBatchOp.executeOn(GatewaySenderBatchOp.java:72) at org.apache.geode.cache.client.internal.SenderProxy.receiveAckFromReceiver(SenderProxy.java:40) at org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher.readAcknowledgement(GatewaySenderEventRemoteDispatcher.java:102) at org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher$AckReaderThread.run(GatewaySenderEventRemoteDispatcher.java:610) Locked ownable synchronizers: - None "ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5363 daemon prio=5 os_prio=0 tid=0x00007f8f4c844800 nid=0xa10 waiting on condition [0x00007f8ed0cbe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fe3178> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) at org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher.destroyConnection(GatewaySenderEventRemoteDispatcher.java:343) at org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher.stop(GatewaySenderEventRemoteDispatcher.java:798) at org.apache.geode.internal.cache.wan.AbstractGatewaySenderEventProcessor.stopProcessing(AbstractGatewaySenderEventProcessor.java:1155) at org.apache.geode.internal.cache.wan.AbstractGatewaySenderEventProcessor$SenderStopperCallable.call(AbstractGatewaySenderEventProcessor.java:1315) at org.apache.geode.internal.cache.wan.AbstractGatewaySenderEventProcessor$SenderStopperCallable.call(AbstractGatewaySenderEventProcessor.java:1304) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x00000000e0fe3440> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5361 daemon prio=5 os_prio=0 tid=0x00007f8f4c844000 nid=0xa0f waiting on condition [0x00007f8ed0dbf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0ef4128> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5360 daemon prio=5 os_prio=0 tid=0x00007f8f4c843000 nid=0xa0e waiting on condition [0x00007f8ed0fc1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0ef4128> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5359 daemon prio=5 os_prio=0 tid=0x00007f8f4c842800 nid=0xa0d waiting on condition [0x00007f8ed17c7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0ef4128> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ClientMembership Event Invoker" #5357 daemon prio=5 os_prio=0 tid=0x00007f8f14002800 nid=0x9e7 waiting on condition [0x00007f8ed1ac8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fdba28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "poolTimer-ln-1" #5354 daemon prio=5 os_prio=0 tid=0x00007f8f14001800 nid=0x9e0 waiting on condition [0x00007f8ed21cf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fd9990> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread-1873" #5353 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a8800 nid=0x9d9 in Object.wait() [0x00007f8f308f2000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132) - locked <0x00000000e0fdf578> (a org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue) Locked ownable synchronizers: - None "Thread-1872" #5351 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a7800 nid=0x9d8 in Object.wait() [0x00007f8f309f3000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132) - locked <0x00000000e0f533c8> (a org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue) Locked ownable synchronizers: - None "Thread-1871" #5349 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a7000 nid=0x9d7 in Object.wait() [0x00007f8f30ef8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132) - locked <0x00000000e0fe1560> (a org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue) Locked ownable synchronizers: - None "Thread-1870" #5347 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a4800 nid=0x9d6 in Object.wait() [0x00007f8f30ff9000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132) - locked <0x00000000e0fdbb28> (a org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue) Locked ownable synchronizers: - None "Thread-1869" #5345 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a4000 nid=0x9d5 in Object.wait() [0x00007f8f312fc000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132) - locked <0x00000000e0feb0f8> (a org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue) Locked ownable synchronizers: - None "Lock Grantor for org.apache.geode.internal.cache.wan.GatewaySenderAdvisor_ln" #5342 daemon prio=5 os_prio=0 tid=0x00007f8f4c848800 nid=0x9d4 in Object.wait() [0x00007f8fad1e3000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.distributed.internal.locks.DLockGrantor$DLockGrantorThread.run(DLockGrantor.java:3592) - locked <0x00000000e0fed0e0> (a java.lang.Object) Locked ownable synchronizers: - None "Lock Grantor for gatewayEventIdIndexMetaData_lockService" #5338 daemon prio=5 os_prio=0 tid=0x00007f8f4c847800 nid=0x9cb in Object.wait() [0x00007f8ed0ec0000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.distributed.internal.locks.DLockGrantor$DLockGrantorThread.run(DLockGrantor.java:3592) - locked <0x00000000e0fe15c8> (a java.lang.Object) Locked ownable synchronizers: - None "Asynchronous disk writer for region ln" #5336 daemon prio=5 os_prio=0 tid=0x00007f8f4c5b3800 nid=0x9c8 in Object.wait() [0x00007f8ed2eda000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:460) at java.util.concurrent.TimeUnit.timedWait(TimeUnit.java:348) at org.apache.geode.internal.cache.DiskStoreImpl$FlusherThread.waitUntilFlushIsReady(DiskStoreImpl.java:1648) - locked <0x00000000e0fdbb90> (a java.lang.Object) at org.apache.geode.internal.cache.DiskStoreImpl$FlusherThread.run(DiskStoreImpl.java:1680) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "P2P message reader for 10.118.32.97(4449)<v4>:32777 shared unordered uid=225 port=40671" #5333 daemon prio=10 os_prio=0 tid=0x00007f8eb8007800 nid=0x99d runnable [0x00007f8e0f1f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:192) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00000000e0fdf628> (a java.lang.Object) at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823) at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "P2P message reader for 10.118.32.97(4344)<v2>:32775 shared ordered uid=234 port=40668" #5332 daemon prio=10 os_prio=0 tid=0x00007f8eb8006800 nid=0x974 runnable [0x00007f8facae0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:192) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00000000e0fe1618> (a java.lang.Object) at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823) at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "P2P message reader for 10.118.32.97(4396)<v3>:32776 shared unordered uid=225 port=40662" #5330 daemon prio=10 os_prio=0 tid=0x00007f8eb8005800 nid=0x941 runnable [0x00007f8e0f2f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:192) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00000000e0fef5c8> (a java.lang.Object) at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823) at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Notification Handler" #5329 daemon prio=10 os_prio=0 tid=0x00007f8e4801e000 nid=0x925 waiting on condition [0x00007f8e0fcfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fdf3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-171" #5327 daemon prio=5 os_prio=0 tid=0x00007f8ef4001800 nid=0x911 in Object.wait() [0x00007f8fad2e4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000000e0ff3510> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "P2P message reader for 10.118.32.97(4344)<v2>:32775 shared unordered uid=231 port=40646" #5326 daemon prio=10 os_prio=0 tid=0x00007f8eb8004800 nid=0x90f runnable [0x00007f8ed2cd8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:192) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00000000e0feb190> (a java.lang.Object) at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823) at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "View Message Processor" #5325 daemon prio=5 os_prio=0 tid=0x00007f8e34003800 nid=0x90c waiting on condition [0x00007f8f307f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fdbbb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604) at org.apache.geode.distributed.internal.DistributionManager$3$1.run(DistributionManager.java:829) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Geode UDP INT-2,poland-54352" #5324 daemon prio=5 os_prio=0 tid=0x00007f8e34002800 nid=0x909 waiting on condition [0x00007f8ed26d4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0efd148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Management Task" #5323 daemon prio=5 os_prio=0 tid=0x00007f8f4c7d4000 nid=0x8f4 waiting on condition [0x00007f8ed13c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e11b7638> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.geode.management.internal.LocalManager$1$1.run(LocalManager.java:128) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Pooled High Priority Message Processor 1" #5322 daemon prio=10 os_prio=0 tid=0x00007f8de0001000 nid=0x8f3 waiting on condition [0x00007f8ed39fa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fd9a58> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604) at org.apache.geode.distributed.internal.DistributionManager$5$1.run(DistributionManager.java:900) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "DiskStoreMonitor" #5321 daemon prio=5 os_prio=0 tid=0x00007f8f4c1df800 nid=0x8f2 waiting on condition [0x00007f8f30cf6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e1013e88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-170" #5320 daemon prio=5 os_prio=0 tid=0x00007f8f4c83b000 nid=0x8f1 in Object.wait() [0x00007f8e0f7f8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x00000000e0fef750> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Non-replicate Region Garbage Collector" #5319 daemon prio=5 os_prio=0 tid=0x00007f8f4c7d7000 nid=0x8f0 in Object.wait() [0x00007f8ed38f9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.doSleep(TombstoneService.java:871) - locked <0x00000000e0feb318> (a org.apache.geode.internal.cache.TombstoneService$NonReplicateTombstoneSweeper) at org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.run(TombstoneService.java:838) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Replicate/Partition Region Garbage Collector" #5318 daemon prio=5 os_prio=0 tid=0x00007f8f4c879800 nid=0x8ef in Object.wait() [0x00007f8fac8de000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.doSleep(TombstoneService.java:871) - locked <0x00000000e0fd9ae0> (a org.apache.geode.internal.cache.TombstoneService$ReplicateTombstoneSweeper) at org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.run(TombstoneService.java:838) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-169" #5317 daemon prio=5 os_prio=0 tid=0x00007f8f4c5a6000 nid=0x8ee in Object.wait() [0x00007f8ed30dc000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000000e0fed0f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "GlobalTXTimeoutMonitor" #5316 daemon prio=5 os_prio=0 tid=0x00007f8f4c82c800 nid=0x8ed in Object.wait() [0x00007f8f30bf5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.jta.TransactionManagerImpl$TransactionTimeOutThread.run(TransactionManagerImpl.java:711) - locked <0x00000000e0fdbcd0> (a java.util.Collections$SynchronizedSortedSet) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread-1854 StatSampler" #5314 daemon prio=10 os_prio=0 tid=0x00007f8f4c7fb000 nid=0x8eb in Object.wait() [0x00007f8e0fefd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.geode.internal.statistics.HostStatSampler.delay(HostStatSampler.java:499) - locked <0x00000000e0eee5a0> (a org.apache.geode.internal.statistics.GemFireStatSampler) at org.apache.geode.internal.statistics.HostStatSampler.run(HostStatSampler.java:198) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "CallbackSampler" #5313 daemon prio=5 os_prio=0 tid=0x00007f8f4c7e2800 nid=0x8ea waiting on condition [0x00007f8e0faf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0feb370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "P2P message reader for 10.118.32.97(3769)<ec><v0>:32770 shared unordered uid=149 port=40643" #5312 daemon prio=10 os_prio=0 tid=0x00007f8eb8003800 nid=0x8e9 runnable [0x00007f8ed3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:192) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00000000e0fe17d0> (a java.lang.Object) at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823) at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "P2P-Handshaker poland.gemstone.com/10.118.32.97:0 Thread 0" #5311 daemon prio=10 os_prio=0 tid=0x00007f8f2c004800 nid=0x8e7 waiting on condition [0x00007f8ed02b8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fef978> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "DM-MemberEventInvoker" #5297 daemon prio=5 os_prio=0 tid=0x00007f8f4c1d4800 nid=0x8df waiting on condition [0x00007f8ed12c2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fd9b70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.geode.distributed.internal.DistributionManager$MemberEventInvoker.run(DistributionManager.java:2392) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Geode UDP INT-1,poland-54352" #5308 daemon prio=5 os_prio=0 tid=0x00007f8e34001800 nid=0x8dc waiting on condition [0x00007f8f310fa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0efd148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Geode Heartbeat Sender" #5307 daemon prio=5 os_prio=0 tid=0x00007f8f4c354800 nid=0x8da waiting on condition [0x00007f8f311fb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$3.sendPeriodicHeartbeats(GMSHealthMonitor.java:714) at org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$3.run(GMSHealthMonitor.java:709) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x00000000e0efa1e8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Geode Failure Detection Server thread 0" #5306 daemon prio=5 os_prio=0 tid=0x00007f8f4c87e000 nid=0x8d9 runnable [0x00007f8fac7dd000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.lambda$startTcpServer$16(GMSHealthMonitor.java:663) at org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$$Lambda$7/1935064956.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x00000000e0f327e0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "P2P Listener Thread /10.118.32.97:36713" #5305 daemon prio=10 os_prio=0 tid=0x00007f8f4c1e2800 nid=0x8d8 runnable [0x00007f8e0fbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) - locked <0x00000000e0fed208> (a java.lang.Object) at org.apache.geode.internal.tcp.TCPConduit.run(TCPConduit.java:649) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-168" #5304 daemon prio=5 os_prio=0 tid=0x00007f8f4c880000 nid=0x8d7 in Object.wait() [0x00007f8e0f4f5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x00000000e0fd9c08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Geode Failure Detection Scheduler" #5303 daemon prio=5 os_prio=0 tid=0x00007f8f4c8fb000 nid=0x8d6 waiting on condition [0x00007f8f305ef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0f32558> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "unicast receiver,poland-54352" #5302 daemon prio=5 os_prio=0 tid=0x00007f8f4c83a000 nid=0x8d5 runnable [0x00007f8f302ee000] java.lang.Thread.State: RUNNABLE at java.net.PlainDatagramSocketImpl.receive0(Native Method) - locked <0x00000000e0f1f930> (a java.net.PlainDatagramSocketImpl) at java.net.AbstractPlainDatagramSocketImpl.receive(AbstractPlainDatagramSocketImpl.java:144) - locked <0x00000000e0f1f930> (a java.net.PlainDatagramSocketImpl) at java.net.DatagramSocket.receive(DatagramSocket.java:812) - locked <0x00000000e0fdbe08> (a java.net.DatagramPacket) - locked <0x00000000e0f1f8e0> (a java.net.MulticastSocket) at org.jgroups.protocols.UDP$PacketReceiver.run(UDP.java:694) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "TransferQueueBundler,poland-54352" #5301 daemon prio=5 os_prio=0 tid=0x00007f8f4c1d1000 nid=0x8d4 waiting on condition [0x00007f8ed1bc9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0eff380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.jgroups.protocols.TransferQueueBundler.run(TransferQueueBundler.java:75) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Geode UDP Timer-2,poland-54352" #5300 daemon prio=5 os_prio=0 tid=0x00007f8f4c81d800 nid=0x8d3 waiting on condition [0x00007f8ed3ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0efe350> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer runner-1,poland-54352" #5299 daemon prio=5 os_prio=0 tid=0x00007f8f4c5a3800 nid=0x8d2 waiting on condition [0x00007f8ed24d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0efef88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.take(DelayQueue.java:223) at java.util.concurrent.DelayQueue.take(DelayQueue.java:70) at org.jgroups.util.TimeScheduler3.run(TimeScheduler3.java:166) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Geode Membership Timer" #5298 daemon prio=5 os_prio=0 tid=0x00007f8f4c1d6000 nid=0x8d1 in Object.wait() [0x00007f8ed2dd9000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x00000000e0f31548> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Function Execution Processor0" #5296 daemon prio=5 os_prio=0 tid=0x00007f8f4c5b1800 nid=0x8d0 waiting on condition [0x00007f8ed01b7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0ff3570> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104) at org.apache.geode.distributed.internal.FunctionExecutionPooledExecutor$2.run(FunctionExecutionPooledExecutor.java:143) at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604) at org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1049) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "PartitionedRegion Message Processor0" #5295 daemon prio=5 os_prio=0 tid=0x00007f8f4c1d2800 nid=0x8cf waiting on condition [0x00007f8ed16c4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fef3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104) at org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92) at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604) at org.apache.geode.distributed.internal.DistributionManager$8$1.run(DistributionManager.java:1006) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "PrMetaData cleanup Message Processor 0" #5294 daemon prio=5 os_prio=0 tid=0x00007f8f4c885000 nid=0x8ce waiting on condition [0x00007f8ed34e0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0feb438> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104) at org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92) at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604) at org.apache.geode.distributed.internal.DistributionManager$7$1.run(DistributionManager.java:967) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Pooled High Priority Message Processor 0" #5293 daemon prio=5 os_prio=0 tid=0x00007f8f4c83c000 nid=0x8cd waiting on condition [0x00007f8f30af4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fd9c20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104) at org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92) at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604) at org.apache.geode.distributed.internal.DistributionManager$5$1.run(DistributionManager.java:900) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Pooled Message Processor 0" #5292 daemon prio=5 os_prio=0 tid=0x00007f8f4c88a000 nid=0x8cc waiting on condition [0x00007f8e0fdfc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e100e968> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104) at org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92) at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604) at org.apache.geode.distributed.internal.DistributionManager$4$1.run(DistributionManager.java:863) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Queued Gateway Listener Thread" #4223 daemon prio=10 os_prio=0 tid=0x00007f8e3c00a800 nid=0x70eb waiting on condition [0x00007f8ed28d6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0d905e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Event Processor for GatewaySender_ln.4" #4220 daemon prio=5 os_prio=0 tid=0x00007f8ed8006800 nid=0x70ea in Object.wait() [0x00007f8f301ed000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486) - locked <0x00000000e0d89fe8> (a java.lang.Object) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203) Locked ownable synchronizers: - None "Event Processor for GatewaySender_ln.3" #4218 daemon prio=5 os_prio=0 tid=0x00007f8ed8005000 nid=0x70e9 in Object.wait() [0x00007f8ed2bd7000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486) - locked <0x00000000e0d89fe8> (a java.lang.Object) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203) Locked ownable synchronizers: - None "Event Processor for GatewaySender_ln.2" #4214 daemon prio=5 os_prio=0 tid=0x00007f8ed8004000 nid=0x70e8 in Object.wait() [0x00007f8ed3dfc000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486) - locked <0x00000000e0d89fe8> (a java.lang.Object) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203) Locked ownable synchronizers: - None "Event Processor for GatewaySender_ln.1" #4211 daemon prio=5 os_prio=0 tid=0x00007f8ed8001800 nid=0x70e7 in Object.wait() [0x00007f8ed1cca000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486) - locked <0x00000000e0d89fe8> (a java.lang.Object) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203) Locked ownable synchronizers: - None "Event Processor for GatewaySender_ln.0" #4208 daemon prio=5 os_prio=0 tid=0x00007f8ed8002800 nid=0x70e6 in Object.wait() [0x00007f8ed1fcd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486) - locked <0x00000000e0d89fe8> (a java.lang.Object) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203) Locked ownable synchronizers: - None "Event Processor for GatewaySender_ln" #4207 daemon prio=5 os_prio=0 tid=0x00007f8f00009800 nid=0x70e5 in Object.wait() [0x00007f8ed1ecc000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1245) - locked <0x00000000e0d8be80> (a org.apache.geode.internal.cache.wan.serial.RemoteSerialGatewaySenderEventProcessor) at java.lang.Thread.join(Thread.java:1319) at org.apache.geode.internal.cache.wan.serial.ConcurrentSerialGatewaySenderEventProcessor.run(ConcurrentSerialGatewaySenderEventProcessor.java:190) Locked ownable synchronizers: - None "Queued Gateway Listener Thread" #4221 daemon prio=10 os_prio=0 tid=0x00007f8e3c003800 nid=0x70e3 waiting on condition [0x00007f8ed22d0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0d90860> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Queued Gateway Listener Thread" #4217 daemon prio=10 os_prio=0 tid=0x00007f8e3c006800 nid=0x70e1 waiting on condition [0x00007f8ed25d3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0d92930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Queued Gateway Listener Thread" #4215 daemon prio=10 os_prio=0 tid=0x00007f8e3c002800 nid=0x70df waiting on condition [0x00007f8f30df7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0d92af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Queued Gateway Listener Thread" #4212 daemon prio=10 os_prio=0 tid=0x00007f8e3c009800 nid=0x70dd waiting on condition [0x00007f8facce2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0d93cc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RMI TCP Connection(1)-10.118.32.97" #27 daemon prio=5 os_prio=0 tid=0x00007f8f54001800 nid=0x10f7 waiting on condition [0x00007f8fad7e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e0fe3420> (a java.util.concurrent.FutureTask) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429) at java.util.concurrent.FutureTask.get(FutureTask.java:191) at java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:244) at org.apache.geode.internal.cache.wan.serial.ConcurrentSerialGatewaySenderEventProcessor.stopProcessing(ConcurrentSerialGatewaySenderEventProcessor.java:296) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderImpl.stop(SerialGatewaySenderImpl.java:134) at org.apache.geode.internal.cache.wan.WANTestBase.stopSender(WANTestBase.java:1635) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest.lambda$testStopOneSerialGatewaySenderBothPrimary$bb17a952$2(SerialGatewaySenderOperationsDUnitTest.java:303) at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest$$Lambda$141/481914353.run(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at hydra.MethExecutor.executeObject(MethExecutor.java:268) at org.apache.geode.test.dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:82) at sun.reflect.GeneratedMethodAccessor309.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323) at sun.rmi.transport.Transport$1.run(Transport.java:200) at sun.rmi.transport.Transport$1.run(Transport.java:197) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Transport.java:196) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$1/385178200.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x00000000e030d318> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00000000e11b70b0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "RMI Reaper" #26 prio=5 os_prio=0 tid=0x00007f90207f5800 nid=0x10f5 in Object.wait() [0x00007f8fad8e6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000e02d4a18> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000000e02d4a18> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at sun.rmi.transport.ObjectTable$Reaper.run(ObjectTable.java:351) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RMI TCP Accept-0" #25 daemon prio=5 os_prio=0 tid=0x00007f90207f1000 nid=0x10f4 runnable [0x00007f8fad9e7000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "GC Daemon" #24 daemon prio=2 os_prio=0 tid=0x00007f90207e0000 nid=0x10f2 in Object.wait() [0x00007f8fadae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at sun.misc.GC$Daemon.run(GC.java:117) - locked <0x00000000e001de88> (a sun.misc.GC$LatencyLock) Locked ownable synchronizers: - None "RMI RenewClean-[10.118.32.97:51910]" #23 daemon prio=5 os_prio=0 tid=0x00007f90207de800 nid=0x10f1 in Object.wait() [0x00007f8fadbe9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000000e0025418> (a java.lang.ref.ReferenceQueue$Lock) at sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:553) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RMI Scheduler(0)" #22 daemon prio=5 os_prio=0 tid=0x00007f90207dc000 nid=0x10f0 waiting on condition [0x00007f8fadcea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e005feb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #19 daemon prio=9 os_prio=0 tid=0x00007f9020116000 nid=0x10ec runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #18 daemon prio=9 os_prio=0 tid=0x00007f9020111000 nid=0x10eb waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #17 daemon prio=9 os_prio=0 tid=0x00007f902010e800 nid=0x10ea waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread9" #16 daemon prio=9 os_prio=0 tid=0x00007f902010c800 nid=0x10e9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread8" #15 daemon prio=9 os_prio=0 tid=0x00007f902010a000 nid=0x10e8 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #14 daemon prio=9 os_prio=0 tid=0x00007f9020108000 nid=0x10e7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #13 daemon prio=9 os_prio=0 tid=0x00007f9020105800 nid=0x10e6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #12 daemon prio=9 os_prio=0 tid=0x00007f9020103800 nid=0x10e5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #11 daemon prio=9 os_prio=0 tid=0x00007f90200f9800 nid=0x10e4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #10 daemon prio=9 os_prio=0 tid=0x00007f90200f7000 nid=0x10e3 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #9 daemon prio=9 os_prio=0 tid=0x00007f90200f5000 nid=0x10e2 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #8 daemon prio=9 os_prio=0 tid=0x00007f90200f3000 nid=0x10e1 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #7 daemon prio=9 os_prio=0 tid=0x00007f90200f0000 nid=0x10e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "JDWP Event Helper Thread" #6 daemon prio=10 os_prio=0 tid=0x00007f90200ed800 nid=0x10df runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "JDWP Transport Listener: dt_socket" #5 daemon prio=10 os_prio=0 tid=0x00007f90200ea000 nid=0x10de runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f90200da800 nid=0x10dd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f90200a3000 nid=0x10dc in Object.wait() [0x00007f8fbc4d6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000000e0013e50> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f90200a1000 nid=0x10db in Object.wait() [0x00007f8fbc5d7000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157) - locked <0x00000000e0014008> (a java.lang.ref.Reference$Lock) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f9020010000 nid=0x10c7 waiting on condition [0x00007f9027646000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.geode.test.dunit.standalone.ChildVM.main(ChildVM.java:58) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f902009c000 nid=0x10da runnable "GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f9020025800 nid=0x10c8 runnable "GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f9020027800 nid=0x10c9 runnable "GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f9020029000 nid=0x10ca runnable "GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f902002b000 nid=0x10cb runnable "GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007f902002d000 nid=0x10cc runnable "GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007f902002e800 nid=0x10cd runnable "GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007f9020030800 nid=0x10ce runnable "GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007f9020032000 nid=0x10cf runnable "GC task thread#8 (ParallelGC)" os_prio=0 tid=0x00007f9020034000 nid=0x10d0 runnable "GC task thread#9 (ParallelGC)" os_prio=0 tid=0x00007f9020036000 nid=0x10d1 runnable "GC task thread#10 (ParallelGC)" os_prio=0 tid=0x00007f9020037800 nid=0x10d2 runnable "GC task thread#11 (ParallelGC)" os_prio=0 tid=0x00007f9020039800 nid=0x10d3 runnable "GC task thread#12 (ParallelGC)" os_prio=0 tid=0x00007f902003b800 nid=0x10d4 runnable "GC task thread#13 (ParallelGC)" os_prio=0 tid=0x00007f902003d000 nid=0x10d5 runnable "GC task thread#14 (ParallelGC)" os_prio=0 tid=0x00007f902003f000 nid=0x10d6 runnable "GC task thread#15 (ParallelGC)" os_prio=0 tid=0x00007f9020041000 nid=0x10d7 runnable "GC task thread#16 (ParallelGC)" os_prio=0 tid=0x00007f9020042800 nid=0x10d8 runnable "GC task thread#17 (ParallelGC)" os_prio=0 tid=0x00007f9020044800 nid=0x10d9 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f9020119800 nid=0x10ed waiting on condition JNI global references: 6409 2016-09-14 09:33:59 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.66-b17 mixed mode): > SerialGatewaySenderOperationsDUnitTest test hangs > ------------------------------------------------- > > Key: GEODE-1894 > URL: https://issues.apache.org/jira/browse/GEODE-1894 > Project: Geode > Issue Type: Bug > Components: wan > Reporter: Hitesh Khamesra > Attachments: threaddump.txt > > > test tries to stop Serial Gateway Sender and that thread just hangs. Event > processors are waiting to become primary. One AckReader thread waiting for > ack. Seems like need to interrupt these threads. Attached thread dump -- This message was sent by Atlassian JIRA (v6.3.4#6332)