[ https://issues.apache.org/jira/browse/IGNITE-9053?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16576299#comment-16576299 ]
Anton Vinogradov commented on IGNITE-9053: ------------------------------------------ Looks like we have deadlock here first thread waits for ack {noformat} "sys-stripe-4-#226264%partitioned.GridCachePartitionedDataStructuresFailoverSelfTest1%" #250984 prio=5 os_prio=0 tid=0x00007f273c018000 nid=0x2de6 waiting on condition [0x00007f274aeee000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at org.apache.ignite.internal.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:177) at org.apache.ignite.internal.util.future.GridFutureAdapter.get(GridFutureAdapter.java:140) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1168) at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:890) at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:85) at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:430) at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:400) at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1079) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:652) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:583) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:464) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:505) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finishDhtLocal(IgniteTxHandler.java:942) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:821) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishRequest(IgniteTxHandler.java:777) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$200(IgniteTxHandler.java:99) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:191) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$3.apply(IgniteTxHandler.java:189) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1056) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:581) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:380) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:306) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:101) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:295) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184) at org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:125) at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1091) at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:496) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) at java.lang.Thread.run(Thread.java:745) {noformat} It sent the event, and node received it, but failed after that. fut can be completed only at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.DiscoveryListener on EVT_NODE_FAILED but EVT_NODE_FAILED can't be handled since we're trying to removeExplicitNodeLocks at previous listener :( {noformat} "disco-event-worker-#226410%partitioned.GridCachePartitionedDataStructuresFailoverSelfTest1%" #251148 prio=5 os_prio=0 tid=0x00007f273c226000 nid=0x2e88 waiting on condition [0x00007f2672d97000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000094e6f670> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:209) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285) at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.lockEntry(GridCacheMapEntry.java:4324) at org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry.removeExplicitNodeLocks(GridDistributedCacheEntry.java:266) at org.apache.ignite.internal.processors.cache.GridCacheMvccManager.removeExplicitNodeLocks(GridCacheMvccManager.java:361) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onNodeLeft(GridDhtPartitionsExchangeFuture.java:3726) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.notifyNodeFail(GridCachePartitionExchangeManager.java:303) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onDiscoveryEvent(GridCachePartitionExchangeManager.java:548) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$200(GridCachePartitionExchangeManager.java:140) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$1.onEvent(GridCachePartitionExchangeManager.java:281) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$DiscoveryListenerWrapper.onEvent(GridEventStorageManager.java:1434) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:873) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:858) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:341) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:307) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:2712) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:2929) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:2741) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) at java.lang.Thread.run(Thread.java:745) {noformat} > testReentrantLockConstantTopologyChangeNonFailoverSafe can hang in case of > broken tx > ------------------------------------------------------------------------------------ > > Key: IGNITE-9053 > URL: https://issues.apache.org/jira/browse/IGNITE-9053 > Project: Ignite > Issue Type: Bug > Components: data structures > Affects Versions: 2.5 > Reporter: Anton Vinogradov > Assignee: Anton Vinogradov > Priority: Major > Labels: MakeTeamcityGreenAgain > Fix For: 2.7 > > > -GridCachePartitionedDataStructuresFailoverSelfTest#testReentrantLockConstantTopologyChangeNonFailoverSafe > -GridCachePartitionedDataStructuresFailoverSelfTest#testCountDownLatchConstantTopologyChange > > can hang in case of broken tx > {noformat} > Pending transactions: > [2018-07-15 14:13:41,210][WARN > ][exchange-worker-#1596354%partitioned.GridCachePartitionedDataStructuresFailoverSelfTest1%][diagnostic] > >>> [txVer=AffinityTopologyVersion [topVer=7, minorTopVer=0], exchWait=true, > tx=GridDhtTxLocal [nearNodeId=1392b1bd-c807-4479-9bfe-fc9f70500000, > nearFutId=14ffca0a461-999e75d0-a333-4bd6-a2a2-7f143d0af773, nearMiniId=1, > nearFinFutId=null, nearFinMiniId=0, nearXidVer=GridCacheVersion > [topVer=143133203, order=1531653200153, nodeOrder=1], > super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], > dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter > [completedBase=null, sndTransformedVals=false, depEnabled=false, > txState=IgniteTxStateImpl [activeCacheIds=[1968300681], recovery=false, > txMap=[IgniteTxEntry [key=KeyCacheObjectImpl [part=494, > val=GridCacheInternalKeyImpl [name=structure, > grpName=default-volatile-ds-group], hasValBytes=true], cacheId=1968300681, > txKey=IgniteTxKey [key=KeyCacheObjectImpl [part=494, > val=GridCacheInternalKeyImpl [name=structure, > grpName=default-volatile-ds-group], hasValBytes=true], cacheId=1968300681], > val=[op=NOOP, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, > val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, > conflictVer=null, explicitVer=null, dhtVer=null, filters=[], > filtersPassed=false, filtersSet=false, entry=GridDhtCacheEntry [rdrs=[], > part=494, super=GridDistributedCacheEntry [super=GridCacheMapEntry > [key=KeyCacheObjectImpl [part=494, val=GridCacheInternalKeyImpl > [name=structure, grpName=default-volatile-ds-group], hasValBytes=true], > val=CacheObjectImpl [val=null, hasValBytes=true], ver=GridCacheVersion > [topVer=143133201, order=1531653200154, nodeOrder=2], hash=2095426867, > extras=GridCacheMvccEntryExtras [mvcc=GridCacheMvcc > [locs=[GridCacheMvccCandidate [nodeId=1bf28b00-feed-412b-a20b-ca9fc1100001, > ver=GridCacheVersion [topVer=143133203, order=1531653200157, nodeOrder=2], > threadId=1947290, id=31143709, topVer=AffinityTopologyVersion [topVer=7, > minorTopVer=0], reentry=null, > otherNodeId=1392b1bd-c807-4479-9bfe-fc9f70500000, otherVer=GridCacheVersion > [topVer=143133203, order=1531653200153, nodeOrder=1], mappedDhtNodes=null, > mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl > [part=494, val=GridCacheInternalKeyImpl [name=structure, > grpName=default-volatile-ds-group], hasValBytes=true], > masks=local=1|owner=1|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, > prevVer=null, nextVer=null]], rmts=null]], flags=2]]], prepared=0, > locked=false, nodeId=null, locMapped=false, expiryPlc=null, > transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, > xidVer=GridCacheVersion [topVer=143133203, order=1531653200157, > nodeOrder=2]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion > [topVer=143133203, order=1531653200157, nodeOrder=2], writeVer=null, > implicit=false, loc=true, threadId=1947290, startTime=1531653200578, > nodeId=1bf28b00-feed-412b-a20b-ca9fc1100001, startVer=GridCacheVersion > [topVer=143133203, order=1531653200157, nodeOrder=2], endVer=null, > isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, > sysInvalidate=false, sys=true, plc=2, commitVer=null, finalizing=NONE, > invalidParts=null, state=ACTIVE, timedOut=false, > topVer=AffinityTopologyVersion [topVer=7, minorTopVer=0], duration=20632ms, > onePhaseCommit=false], size=1]]]] > {noformat} -- This message was sent by Atlassian JIRA (v7.6.3#76005)