Maxim Muzafarov created IGNITE-12981:
----------------------------------------
Summary: Snapshot operation fails on exchange if coordinator left
the cluster
Key: IGNITE-12981
URL: https://issues.apache.org/jira/browse/IGNITE-12981
Project: Ignite
Issue Type: Bug
Reporter: Maxim Muzafarov
Assignee: Maxim Muzafarov
Fix For: 2.9
Possible steps:
1. Start snapshot operation with {{SnapshotDicoveryMessage}} with triggering
pme-free.
2. Stop the coordinator node.
3. Fails on assertion (SingleMessage should not be sent).
{code}
[2020-05-03
15:05:35,548][ERROR][sys-#94%snapshot.IgniteClusterSnapshotSelfTest1%][IgniteTestResources]
Critical system error detected. Will be handled accordingly to configured
handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
[ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED,
SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext
[type=CRITICAL_ERROR, err=java.lang.AssertionError:
GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryCustomEvent
[customMsg=null, affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1],
super=DiscoveryEvent [evtNode=TcpDiscoveryNode
[id=ed2e276b-79a3-4de3-925a-68d602000000,
consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList
[127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1,
intOrder=1, lastExchangeTime=1588507532715, loc=false,
ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad,
msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]],
crd=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000,
consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList
[127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1,
intOrder=1, lastExchangeTime=1588507532715, loc=false,
ver=2.9.0#20200503-sha1:00000000, isClient=false],
exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=3,
minorTopVer=1], discoEvt=DiscoveryCustomEvent [customMsg=null,
affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1],
super=DiscoveryEvent [evtNode=TcpDiscoveryNode
[id=ed2e276b-79a3-4de3-925a-68d602000000,
consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList
[127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1,
intOrder=1, lastExchangeTime=1588507532715, loc=false,
ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad,
msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]], nodeId=ed2e276b,
evt=DISCOVERY_CUSTOM_EVT], added=true, exchangeType=ALL,
initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true,
hash=1771643357], init=false, lastVer=null,
partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion
[topVer=3, minorTopVer=1], futures=[ExplicitLockReleaseFuture
[topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]],
AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=3,
minorTopVer=1], futures=[]], DataStreamerReleaseFuture
[topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]],
LocalTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1],
futures=[]], AllTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3,
minorTopVer=1], futures=[RemoteTxReleaseFuture [topVer=AffinityTopologyVersion
[topVer=3, minorTopVer=1], futures=[]]]]]], exchActions=null,
affChangeMsg=null, centralizedAff=false, forceAffReassignment=false,
exchangeLocE=null, cacheChangeFailureMsgSent=false, done=true, state=SRV,
registerCachesFuture=null, partitionsSent=false, partitionsReceived=false,
delayedLatestMsg=null, afterLsnrCompleteFut=GridFutureAdapter
[ignoreInterrupts=false, state=DONE, res=null, hash=197787912],
timeBag=o.a.i.i.util.TimeBag@1d97d94c, startTime=1365700834756,
initTime=1588507535439, rebalanced=false, evtLatch=0, remaining=HashSet
[ed2e276b-79a3-4de3-925a-68d602000000], mergedJoinExchMsgs=null,
awaitMergedMsgs=0, super=GridFutureAdapter [ignoreInterrupts=false, state=DONE,
res=AffinityTopologyVersion [topVer=3, minorTopVer=1], hash=530592354]]]]
java.lang.AssertionError: GridDhtPartitionsExchangeFuture
[firstDiscoEvt=DiscoveryCustomEvent [customMsg=null,
affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1],
super=DiscoveryEvent [evtNode=TcpDiscoveryNode
[id=ed2e276b-79a3-4de3-925a-68d602000000,
consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList
[127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1,
intOrder=1, lastExchangeTime=1588507532715, loc=false,
ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad,
msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]],
crd=TcpDiscoveryNode [id=ed2e276b-79a3-4de3-925a-68d602000000,
consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList
[127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1,
intOrder=1, lastExchangeTime=1588507532715, loc=false,
ver=2.9.0#20200503-sha1:00000000, isClient=false],
exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=3,
minorTopVer=1], discoEvt=DiscoveryCustomEvent [customMsg=null,
affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1],
super=DiscoveryEvent [evtNode=TcpDiscoveryNode
[id=ed2e276b-79a3-4de3-925a-68d602000000,
consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList
[127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1,
intOrder=1, lastExchangeTime=1588507532715, loc=false,
ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, nodeId8=f6fa6fad,
msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1588507535430]], nodeId=ed2e276b,
evt=DISCOVERY_CUSTOM_EVT], added=true, exchangeType=ALL,
initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true,
hash=1771643357], init=false, lastVer=null,
partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion
[topVer=3, minorTopVer=1], futures=[ExplicitLockReleaseFuture
[topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]],
AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=3,
minorTopVer=1], futures=[]], DataStreamerReleaseFuture
[topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]],
LocalTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1],
futures=[]], AllTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3,
minorTopVer=1], futures=[RemoteTxReleaseFuture [topVer=AffinityTopologyVersion
[topVer=3, minorTopVer=1], futures=[]]]]]], exchActions=null,
affChangeMsg=null, centralizedAff=false, forceAffReassignment=false,
exchangeLocE=null, cacheChangeFailureMsgSent=false, done=true, state=SRV,
registerCachesFuture=null, partitionsSent=false, partitionsReceived=false,
delayedLatestMsg=null, afterLsnrCompleteFut=GridFutureAdapter
[ignoreInterrupts=false, state=DONE, res=null, hash=197787912],
timeBag=org.apache.ignite.internal.util.TimeBag@1d97d94c,
startTime=1365700834756, initTime=1588507535439, rebalanced=false, evtLatch=0,
remaining=HashSet [ed2e276b-79a3-4de3-925a-68d602000000],
mergedJoinExchMsgs=null, awaitMergedMsgs=0, super=GridFutureAdapter
[ignoreInterrupts=false, state=DONE, res=AffinityTopologyVersion [topVer=3,
minorTopVer=1], hash=530592354]]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$3.apply(GridDhtPartitionsExchangeFuture.java:2966)
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$3.apply(GridDhtPartitionsExchangeFuture.java:2949)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:354)
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.waitAndReplyToNode(GridDhtPartitionsExchangeFuture.java:2949)
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processSingleMessage(GridDhtPartitionsExchangeFuture.java:3005)
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$100(GridDhtPartitionsExchangeFuture.java:151)
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2908)
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2896)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:354)
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveSingleMessage(GridDhtPartitionsExchangeFuture.java:2896)
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processSinglePartitionUpdate(GridCachePartitionExchangeManager.java:1999)
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.preprocessSingleMessage(GridCachePartitionExchangeManager.java:532)
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$1100(GridCachePartitionExchangeManager.java:182)
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:415)
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:397)
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3705)
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3684)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
