Alexey Goncharuk created IGNITE-11050:
-----------------------------------------
Summary: Potential deadlock caused by DhtColocatedLockFuture#map
being called inside topology read lock
Key: IGNITE-11050
URL: https://issues.apache.org/jira/browse/IGNITE-11050
Project: Ignite
Issue Type: Bug
Reporter: Alexey Goncharuk
I observed the following stacktrace on TC during tests analysis:
{code}
Thread
[name="exchange-worker-#18471%near.GridCachePartitionedNodeRestartTest0%",
id=23715, state=WAITING, blockCnt=860, waitCnt=775]
Lock
[object=java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2bfb6b49,
ownerName=null, ownerId=-1]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
at
java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
at
o.a.i.i.util.StripedCompositeReadWriteLock$WriteLock.lock0(StripedCompositeReadWriteLock.java:173)
at
o.a.i.i.util.StripedCompositeReadWriteLock$WriteLock.lock(StripedCompositeReadWriteLock.java:142)
at
o.a.i.i.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition0(GridDhtPartitionTopologyImpl.java:925)
at
o.a.i.i.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition(GridDhtPartitionTopologyImpl.java:826)
at
o.a.i.i.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.localPartition(GridCachePartitionedConcurrentMap.java:70)
at
o.a.i.i.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.putEntryIfObsoleteOrAbsent(GridCachePartitionedConcurrentMap.java:89)
at
o.a.i.i.processors.cache.GridCacheAdapter.entryEx(GridCacheAdapter.java:1019)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtCacheAdapter.entryEx(GridDhtCacheAdapter.java:544)
at
o.a.i.i.processors.cache.transactions.IgniteTxManager.txUnlock(IgniteTxManager.java:1764)
at
o.a.i.i.processors.cache.transactions.IgniteTxManager.unlockMultiple(IgniteTxManager.java:1775)
at
o.a.i.i.processors.cache.transactions.IgniteTxManager.rollbackTx(IgniteTxManager.java:1347)
at
o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userRollback(IgniteTxLocalAdapter.java:1075)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxLocal.localFinish(GridNearTxLocal.java:3602)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxFinishFuture.doFinish(GridNearTxFinishFuture.java:440)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:390)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3833)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3784)
at
o.a.i.i.processors.cache.GridCacheAdapter$53.applyx(GridCacheAdapter.java:4409)
at
o.a.i.i.processors.cache.GridCacheAdapter$53.applyx(GridCacheAdapter.java:4399)
at o.a.i.i.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38)
at
o.a.i.i.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78)
at
o.a.i.i.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70)
at
o.a.i.i.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478)
at
o.a.i.i.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:81)
at
o.a.i.i.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70)
at
o.a.i.i.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478)
at
o.a.i.i.util.future.GridEmbeddedFuture$AsyncListener1.apply(GridEmbeddedFuture.java:298)
at
o.a.i.i.util.future.GridEmbeddedFuture$AsyncListener1.apply(GridEmbeddedFuture.java:285)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
at
o.a.i.i.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
at
o.a.i.i.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.onComplete(GridDhtColocatedLockFuture.java:647)
at
o.a.i.i.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.onDone(GridDhtColocatedLockFuture.java:618)
at
o.a.i.i.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.map(GridDhtColocatedLockFuture.java:916)
at
o.a.i.i.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.mapOnTopology(GridDhtColocatedLockFuture.java:873)
at
o.a.i.i.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.lambda$mapOnTopology$27f50bf2$1(GridDhtColocatedLockFuture.java:886)
at
o.a.i.i.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture$$Lambda$166/1888916441.apply(Unknown
Source)
at
o.a.i.i.processors.timeout.GridTimeoutProcessor$2.apply(GridTimeoutProcessor.java:181)
at
o.a.i.i.processors.timeout.GridTimeoutProcessor$2.apply(GridTimeoutProcessor.java:173)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2214)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.finishExchangeOnCoordinator(GridDhtPartitionsExchangeFuture.java:3499)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onAllReceived(GridDhtPartitionsExchangeFuture.java:3268)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processSingleMessage(GridDhtPartitionsExchangeFuture.java:2883)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$100(GridDhtPartitionsExchangeFuture.java:144)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2690)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2678)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:467)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.initDone(GridDhtPartitionsExchangeFuture.java:4316)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.distributedExchange(GridDhtPartitionsExchangeFuture.java:1506)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.init(GridDhtPartitionsExchangeFuture.java:833)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:2886)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:2735)
at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
{code}
As one can see, {{GridDhtColocatedLockFuture#map(Collection<KeyCacheObject>
keys, boolean remap, boolean topLocked)}} is called inside topology read lock,
then a chain of future notifications leads to a transaction rollback, an entry
creation and an attempt to acquire topology write lock, which results in a
deadlock.
{{map}} should be called outside of topology read lock, the same way as it is
done for other futures.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
