Maxim Muzafarov created IGNITE-12981: ----------------------------------------
Summary: Snapshot operation fails on exchange if coordinator left the cluster Key: IGNITE-12981 URL: https://issues.apache.org/jira/browse/IGNITE-12981 Project: Ignite Issue Type: Bug Reporter: Maxim Muzafarov Assignee: Maxim Muzafarov Fix For: 2.9 Possible steps: 1. Start snapshot operation with {{SnapshotDicoveryMessage}} with triggering pme-free. 2. Stop the coordinator node. 3. Fails on assertion (SingleMessage should not be sent). {code} [2020-05-03 15:05:35,548][ERROR][sys-#94%snapshot.IgniteClusterSnapshotSelfTest1%][IgniteTestResources] Critical system error detected. Will be handled accordingly to configured handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=java.lang.AssertionError: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryCustomEvent [customMsg=null, affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], super=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000, consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1588507532715, loc=false, ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]], crd=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000, consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1588507532715, loc=false, ver=2.9.0#20200503-sha1:00000000, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], discoEvt=DiscoveryCustomEvent [customMsg=null, affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], super=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000, consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1588507532715, loc=false, ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]], nodeId=ed2e276b, evt=DISCOVERY_CUSTOM_EVT], added=true, exchangeType=ALL, initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true, hash=1771643357], init=false, lastVer=null, partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], LocalTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], AllTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[RemoteTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]]]]]], exchActions=null, affChangeMsg=null, centralizedAff=false, forceAffReassignment=false, exchangeLocE=null, cacheChangeFailureMsgSent=false, done=true, state=SRV, registerCachesFuture=null, partitionsSent=false, partitionsReceived=false, delayedLatestMsg=null, afterLsnrCompleteFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=null, hash=197787912], timeBag=o.a.i.i.util.TimeBag@1d97d94c, startTime=1365700834756, initTime=1588507535439, rebalanced=false, evtLatch=0, remaining=HashSet [ed2e276b-79a3-4de3-925a-68d602000000], mergedJoinExchMsgs=null, awaitMergedMsgs=0, super=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=AffinityTopologyVersion [topVer=3, minorTopVer=1], hash=530592354]]]] java.lang.AssertionError: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryCustomEvent [customMsg=null, affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], super=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000, consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1588507532715, loc=false, ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]], crd=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000, consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1588507532715, loc=false, ver=2.9.0#20200503-sha1:00000000, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], discoEvt=DiscoveryCustomEvent [customMsg=null, affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], super=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000, consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1588507532715, loc=false, ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]], nodeId=ed2e276b, evt=DISCOVERY_CUSTOM_EVT], added=true, exchangeType=ALL, initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true, hash=1771643357], init=false, lastVer=null, partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], LocalTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], AllTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[RemoteTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]]]]]], exchActions=null, affChangeMsg=null, centralizedAff=false, forceAffReassignment=false, exchangeLocE=null, cacheChangeFailureMsgSent=false, done=true, state=SRV, registerCachesFuture=null, partitionsSent=false, partitionsReceived=false, delayedLatestMsg=null, afterLsnrCompleteFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=null, hash=197787912], timeBag=org.apache.ignite.internal.util.TimeBag@1d97d94c, startTime=1365700834756, initTime=1588507535439, rebalanced=false, evtLatch=0, remaining=HashSet [ed2e276b-79a3-4de3-925a-68d602000000], mergedJoinExchMsgs=null, awaitMergedMsgs=0, super=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=AffinityTopologyVersion [topVer=3, minorTopVer=1], hash=530592354]] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$3.apply(GridDhtPartitionsExchangeFuture.java:2966) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$3.apply(GridDhtPartitionsExchangeFuture.java:2949) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:354) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.waitAndReplyToNode(GridDhtPartitionsExchangeFuture.java:2949) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processSingleMessage(GridDhtPartitionsExchangeFuture.java:3005) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$100(GridDhtPartitionsExchangeFuture.java:151) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2908) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2896) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:354) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveSingleMessage(GridDhtPartitionsExchangeFuture.java:2896) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processSinglePartitionUpdate(GridCachePartitionExchangeManager.java:1999) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.preprocessSingleMessage(GridCachePartitionExchangeManager.java:532) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$1100(GridCachePartitionExchangeManager.java:182) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:415) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:397) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3705) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3684) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) {code} -- This message was sent by Atlassian Jira (v8.3.4#803005)