[
https://issues.apache.org/jira/browse/IGNITE-13251?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander Lapin reassigned IGNITE-13251:
----------------------------------------
Assignee: Alexander Lapin
> Deadlock between grid-timeout-worker and a thread opening a communication
> connection.
> -------------------------------------------------------------------------------------
>
> Key: IGNITE-13251
> URL: https://issues.apache.org/jira/browse/IGNITE-13251
> Project: Ignite
> Issue Type: Bug
> Reporter: Alexander Lapin
> Assignee: Alexander Lapin
> Priority: Major
> Time Spent: 10m
> Remaining Estimate: 0h
>
> rid-timeout-worker is known to go into a deadlock state with other threads in
> a few scenarios.
> The general scheme is:
> 1. A thread `T` is holding lock `L` and is trying to establish a
> communication connection, hanging in `safeTcpHandshake` method. Due to the
> logic of `safeTcpHandshake`, `grid-timeout-worker` needs to send a signal to
> `T` in order for it to proceed.
> 2. `grid-timeout-worker` is trying to acquire `L`. Hence, the deadlock.
> It may include more threads. The lock `L` can be different: checkpoint lock,
> GridCacheMapEntry lock, etc.
> #
> ## Example
> The below example shows a lock between
> * `grid-timeout-worker` trying to acquire a cp read lock in a
> `dumpLongRunningTransactions`
> * `tcp-comm-worker` trying to establish a connection but hanging on socket
> read due to unstable network
> * checkpointer trying to start a checkpoint and acquire cp write lock
> * `utility` worker waiting for the connection to be established by
> `tcp-comm-worker` while holding cp read lock
> {code:java}
> Thread [name="grid-timeout-worker-#23", id=42, state=WAITING, blockCnt=6991,
> waitCnt=1746467]
> Lock
> [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@4545a8b9,
> ownerName=null, ownerId=-1]
> at sun.misc.Unsafe.park(Native Method)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283)
> at
> java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727)
> at
> o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.checkpointReadLock(GridCacheDatabaseSharedManager.java:1707)
> at
> o.a.i.i.processors.cache.distributed.dht.GridPartitionedSingleGetFuture.setResult(GridPartitionedSingleGetFuture.java:715)
> at
> o.a.i.i.processors.cache.distributed.dht.GridPartitionedSingleGetFuture.localGet(GridPartitionedSingleGetFuture.java:511)
> at
> o.a.i.i.processors.cache.distributed.dht.GridPartitionedSingleGetFuture.tryLocalGet(GridPartitionedSingleGetFuture.java:399)
> at
> o.a.i.i.processors.cache.distributed.dht.GridPartitionedSingleGetFuture.mapKeyToNode(GridPartitionedSingleGetFuture.java:366)
> at
> o.a.i.i.processors.cache.distributed.dht.GridPartitionedSingleGetFuture.map(GridPartitionedSingleGetFuture.java:243)
> at
> o.a.i.i.processors.cache.distributed.dht.GridPartitionedSingleGetFuture.init(GridPartitionedSingleGetFuture.java:232)
> at
> o.a.i.i.processors.cache.distributed.dht.colocated.GridDhtColocatedCache.getAsync(GridDhtColocatedCache.java:246)
> at
> o.a.i.i.processors.cache.GridCacheAdapter.get0(GridCacheAdapter.java:4190)
> at
> o.a.i.i.processors.cache.GridCacheAdapter.get(GridCacheAdapter.java:4171)
> at
> o.a.i.i.processors.cache.GridCacheAdapter.get(GridCacheAdapter.java:1362)
> at
> o.a.i.i.processors.task.GridTaskProcessor.saveTaskMetadata(GridTaskProcessor.java:908)
> at
> o.a.i.i.processors.task.GridTaskProcessor.startTask(GridTaskProcessor.java:746)
> at
> o.a.i.i.processors.task.GridTaskProcessor.execute(GridTaskProcessor.java:477)
> at
> o.a.i.i.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:674)
> at
> o.a.i.i.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:479)
> at o.a.i.i.IgniteComputeImpl.callAsync0(IgniteComputeImpl.java:809)
> at o.a.i.i.IgniteComputeImpl.callAsync(IgniteComputeImpl.java:794)
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager.dumpLongRunningTransaction(GridCachePartitionExchangeManager.java:2115)
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager.dumpLongRunningOperations0(GridCachePartitionExchangeManager.java:2012)
> - locked
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ActionLimiter@47b86e8f
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager.dumpLongRunningOperations(GridCachePartitionExchangeManager.java:2180)
> at o.a.i.i.IgniteKernal$4.run(IgniteKernal.java:1478)
> at
> o.a.i.i.processors.timeout.GridTimeoutProcessor$CancelableTask.onTimeout(GridTimeoutProcessor.java:410)
> - locked
> o.a.i.i.processors.timeout.GridTimeoutProcessor$CancelableTask@67c951c7
> at
> o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:279)
> at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120)
> at java.lang.Thread.run(Thread.java:748)
> Thread [name="tcp-comm-worker-#1", id=50, state=RUNNABLE, blockCnt=16,
> waitCnt=6712]
> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
> at sun.nio.ch.IOUtil.read(IOUtil.java:197)
> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:377)
> - locked java.lang.Object@71681f0c
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.safeTcpHandshake(TcpCommunicationSpi.java:3906)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3420)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.createNioClient(TcpCommunicationSpi.java:3055)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2908)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.access$6300(TcpCommunicationSpi.java:272)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.processDisconnect(TcpCommunicationSpi.java:4603)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.body(TcpCommunicationSpi.java:4408)
> at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi$7.body(TcpCommunicationSpi.java:2261)
> at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
> Thread [name="db-checkpoint-thread-#69", id=144, state=WAITING, blockCnt=0,
> waitCnt=12657468]
> Lock
> [object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@4545a8b9,
> ownerName=null, ownerId=-1]
> at sun.misc.Unsafe.park(Native Method)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
> at
> java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
> at
> o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.markCheckpointBegin(GridCacheDatabaseSharedManager.java:4059)
> at
> o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.doCheckpoint(GridCacheDatabaseSharedManager.java:3637)
> at
> o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.body(GridCacheDatabaseSharedManager.java:3522)
> at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120)
> at java.lang.Thread.run(Thread.java:748)
> Thread [name="utility-#53812", id=59375, state=TIMED_WAITING, blockCnt=0,
> waitCnt=1433]
> at sun.misc.Unsafe.park(Native Method)
> at
> java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338)
> at
> o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:219)
> at
> o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:160)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2959)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2751)
> at
> o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2710)
> at
> o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:1643)
> at
> o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:1863)
> at
> o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1873)
> at
> o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1844)
> at
> o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1826)
> at
> o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1244)
> at
> o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1181)
> at
> o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:882)
> at
> o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:85)
> at
> o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:429)
> at
> o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:448)
> at
> o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1115)
> at
> o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:747)
> at
> o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:796)
> at
> o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:603)
> at
> o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:475)
> at
> o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:532)
> at
> o.a.i.i.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:1042)
> at
> o.a.i.i.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:921)
> at
> o.a.i.i.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:877)
> at
> o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:109)
> at
> o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:203)
> at
> o.a.i.i.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:201)
> at
> o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1078)
> at
> o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:587)
> at
> o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:386)
> at
> o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:312)
> at
> o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:102)
> at
> o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:301)
> at
> o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
> at
> o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
> at
> o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
> at
> o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> {code}
> {color:#172b4d}{{}}{color}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
