Hi, I am getting the following error in Ignite version 1.8.1 :
"Failed to wait for partition release future" The related logs lines are at the bottom. I was curious if it could be related to https://issues.apache.org/jira/browse/IGNITE-3212 which has fix for version 1.9. Could this be the case ? Otherwise, is this likely to be a bug in ignite, or something specific to my environment? Cheers Cameron - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287661 WARN o.a.i.i.p.c.d.d.p.GridDhtPartitionsExchangeFuture [exchange-worker-#26%webapp%] Failed to wait for partition release future [topVer=AffinityTopologyVersion [topVer=21, minorTopVer=2], node=79d74dfa-9903-42e3-af94-524ab08c1f97]. Dumping pending objects that might be the cause: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287661 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Ready affinity version: AffinityTopologyVersion [topVer=21, minorTopVer=1] - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287667 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Last exchange future: GridDhtPartitionsExchangeFuture [dummy=false, forcePreload=false, reassign=false, discoEvt=DiscoveryCustomEvent [customMsg=DynamicCacheChangeBatch [reqs=[DynamicCacheChangeRequest [deploymentId=3f757f24a51-ee74fcc5-5cea-4548-9cd6-8a201c29fdf9, startCfg=CacheConfiguration [name=userTable.cmsDataVehicles.query, storeConcurrentLoadAllThreshold=5, rebalancePoolSize=2, rebalanceTimeout=10000, evictPlc=null, evictSync=false, evictKeyBufSize=1024, evictSyncConcurrencyLvl=4, evictSyncTimeout=10000, evictFilter=null, evictMaxOverflowRatio=10.0, eagerTtl=true, dfltLockTimeout=0, startSize=1500000, nearCfg=null, writeSync=PRIMARY_SYNC, storeFactory=null, storeKeepBinary=false, loadPrevVal=false, aff=o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction@7e1c4f4, cacheMode=REPLICATED, atomicityMode=ATOMIC, atomicWriteOrderMode=PRIMARY, backups=2147483647, invalidate=false, tmLookupClsName=null, rebalanceMode=ASYNC, rebalanceOrder=0, rebalanceBatchSize=524288, rebalanceBatchesPrefetchCount=2, offHeapMaxMem=-1, swapEnabled=false, maxConcurrentAsyncOps=500, writeBehindEnabled=false, writeBehindFlushSize=10240, writeBehindFlushFreq=5000, writeBehindFlushThreadCnt=1, writeBehindBatchSize=512, maxQryIterCnt=1024, memMode=ONHEAP_TIERED, affMapper=o.a.i.i.processors.cache.CacheDefaultBinaryAffinityKeyMapper@6ca896dd, rebalanceDelay=0, rebalanceThrottle=0, interceptor=null, longQryWarnTimeout=3000, qryDetailMetricsSz=0, readFromBackup=true, nodeFilter=o.a.i.configuration.CacheConfiguration$IgniteAllNodesPredicate@db2015b, sqlSchema=null, sqlEscapeAll=false, sqlOnheapRowCacheSize=10240, snapshotableIdx=false, cpOnRead=true, topValidator=null], cacheType=USER, initiatingNodeId=79d74dfa-9903-42e3-af94-524ab08c1f97, nearCacheCfg=null, clientStartOnly=false, stop=false, close=false, failIfExists=true, template=false, rcvdFrom=null, exchangeNeeded=true, cacheFutTopVer=null, cacheName=userTable.cmsDataVehicles.query]], clientNodes=null, id=4f757f24a51-ee74fcc5-5cea-4548-9cd6-8a201c29fdf9, clientReconnect=false], affTopVer=AffinityTopologyVersion [topVer=21, minorTopVer=2], super=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=79d74dfa-9903-42e3-af94-524ab08c1f97, addrs=[0:0:0:0:0:0:0:1%lo, 10.130.1.43, 127.0.0.1], sockAddrs=[webapp-92-50ik3/10.130.1.43:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=21, intOrder=12, lastExchangeTime=1487193806549, loc=true, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], topVer=21, nodeId8=79d74dfa, msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1487193798024]], crd=TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=21, minorTopVer=2], nodeId=79d74dfa, evt=DISCOVERY_CUSTOM_EVT], added=true, initFut=GridFutureAdapter [resFlag=0, res=null, startTime=1487193798024, endTime=0, ignoreInterrupts=false, state=INIT], init=false, topSnapshot=null, lastVer=null, partReleaseFut=GridCompoundFuture [rdc=null, initFlag=1, lsnrCalls=3, done=false, cancelled=false, err=null, futs=[true, true, false, true]], affChangeMsg=null, skipPreload=false, clientOnlyExchange=false, initTs=1487193798024, centralizedAff=false, evtLatch=0, remaining=[0940d968-a901-4c78-8354-8171628d41aa, 659ea029-ca80-4413-af47-67b446ffd3e7], srvNodes=[TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=0940d968-a901-4c78-8354-8171628d41aa, addrs=[0:0:0:0:0:0:0:1%lo, 10.129.1.129, 127.0.0.1], sockAddrs=[/10.129.1.129:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=20, intOrder=11, lastExchangeTime=1487178534516, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=79d74dfa-9903-42e3-af94-524ab08c1f97, addrs=[0:0:0:0:0:0:0:1%lo, 10.130.1.43, 127.0.0.1], sockAddrs=[webapp-92-50ik3/10.130.1.43:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=21, intOrder=12, lastExchangeTime=1487193806549, loc=true, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false]], super=GridFutureAdapter [resFlag=0, res=null, startTime=1487193798024, endTime=0, ignoreInterrupts=false, state=INIT]] - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287667 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Pending exchange futures: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287668 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Last 10 exchange futures (total: 3): - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287668 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] >>> GridDhtPartitionsExchangeFuture [dummy=false, forcePreload=false, reassign=false, discoEvt=DiscoveryCustomEvent [customMsg=DynamicCacheChangeBatch [reqs=[DynamicCacheChangeRequest [deploymentId=3f757f24a51-ee74fcc5-5cea-4548-9cd6-8a201c29fdf9, startCfg=CacheConfiguration [name=userTable.cmsDataVehicles.query, storeConcurrentLoadAllThreshold=5, rebalancePoolSize=2, rebalanceTimeout=10000, evictPlc=null, evictSync=false, evictKeyBufSize=1024, evictSyncConcurrencyLvl=4, evictSyncTimeout=10000, evictFilter=null, evictMaxOverflowRatio=10.0, eagerTtl=true, dfltLockTimeout=0, startSize=1500000, nearCfg=null, writeSync=PRIMARY_SYNC, storeFactory=null, storeKeepBinary=false, loadPrevVal=false, aff=o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction@7e1c4f4, cacheMode=REPLICATED, atomicityMode=ATOMIC, atomicWriteOrderMode=PRIMARY, backups=2147483647, invalidate=false, tmLookupClsName=null, rebalanceMode=ASYNC, rebalanceOrder=0, rebalanceBatchSize=524288, rebalanceBatchesPrefetchCount=2, offHeapMaxMem=-1, swapEnabled=false, maxConcurrentAsyncOps=500, writeBehindEnabled=false, writeBehindFlushSize=10240, writeBehindFlushFreq=5000, writeBehindFlushThreadCnt=1, writeBehindBatchSize=512, maxQryIterCnt=1024, memMode=ONHEAP_TIERED, affMapper=o.a.i.i.processors.cache.CacheDefaultBinaryAffinityKeyMapper@6ca896dd, rebalanceDelay=0, rebalanceThrottle=0, interceptor=null, longQryWarnTimeout=3000, qryDetailMetricsSz=0, readFromBackup=true, nodeFilter=o.a.i.configuration.CacheConfiguration$IgniteAllNodesPredicate@db2015b, sqlSchema=null, sqlEscapeAll=false, sqlOnheapRowCacheSize=10240, snapshotableIdx=false, cpOnRead=true, topValidator=null], cacheType=USER, initiatingNodeId=79d74dfa-9903-42e3-af94-524ab08c1f97, nearCacheCfg=null, clientStartOnly=false, stop=false, close=false, failIfExists=true, template=false, rcvdFrom=null, exchangeNeeded=true, cacheFutTopVer=null, cacheName=userTable.cmsDataVehicles.query]], clientNodes=null, id=4f757f24a51-ee74fcc5-5cea-4548-9cd6-8a201c29fdf9, clientReconnect=false], affTopVer=AffinityTopologyVersion [topVer=21, minorTopVer=2], super=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=79d74dfa-9903-42e3-af94-524ab08c1f97, addrs=[0:0:0:0:0:0:0:1%lo, 10.130.1.43, 127.0.0.1], sockAddrs=[webapp-92-50ik3/10.130.1.43:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=21, intOrder=12, lastExchangeTime=1487193806549, loc=true, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], topVer=21, nodeId8=79d74dfa, msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1487193798024]], crd=TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=21, minorTopVer=2], nodeId=79d74dfa, evt=DISCOVERY_CUSTOM_EVT], added=true, initFut=GridFutureAdapter [resFlag=0, res=null, startTime=1487193798024, endTime=0, ignoreInterrupts=false, state=INIT], init=false, topSnapshot=null, lastVer=null, partReleaseFut=GridCompoundFuture [rdc=null, initFlag=1, lsnrCalls=3, done=false, cancelled=false, err=null, futs=[true, true, false, true]], affChangeMsg=null, skipPreload=false, clientOnlyExchange=false, initTs=1487193798024, centralizedAff=false, evtLatch=0, remaining=[0940d968-a901-4c78-8354-8171628d41aa, 659ea029-ca80-4413-af47-67b446ffd3e7], srvNodes=[TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=0940d968-a901-4c78-8354-8171628d41aa, addrs=[0:0:0:0:0:0:0:1%lo, 10.129.1.129, 127.0.0.1], sockAddrs=[/10.129.1.129:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=20, intOrder=11, lastExchangeTime=1487178534516, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=79d74dfa-9903-42e3-af94-524ab08c1f97, addrs=[0:0:0:0:0:0:0:1%lo, 10.130.1.43, 127.0.0.1], sockAddrs=[webapp-92-50ik3/10.130.1.43:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=21, intOrder=12, lastExchangeTime=1487193806549, loc=true, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false]], super=GridFutureAdapter [resFlag=0, res=null, startTime=1487193798024, endTime=0, ignoreInterrupts=false, state=INIT]] - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287669 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] >>> GridDhtPartitionsExchangeFuture [dummy=false, forcePreload=false, reassign=false, discoEvt=DiscoveryCustomEvent [customMsg=null, affTopVer=AffinityTopologyVersion [topVer=21, minorTopVer=1], super=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], topVer=21, nodeId8=79d74dfa, msg=null, type=DISCOVERY_CUSTOM_EVT, tstamp=1487178539604]], crd=TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=21, minorTopVer=1], nodeId=659ea029, evt=DISCOVERY_CUSTOM_EVT], added=true, initFut=GridFutureAdapter [resFlag=2, res=true, startTime=1487178539604, endTime=1487178539762, ignoreInterrupts=false, state=DONE], init=true, topSnapshot=null, lastVer=null, partReleaseFut=GridCompoundFuture [rdc=null, initFlag=1, lsnrCalls=4, done=true, cancelled=false, err=null, futs=[true, true, true, true]], affChangeMsg=CacheAffinityChangeMessage [id=426bdb24a51-0943897a-92a2-454f-8ce0-2cfc43f893df, topVer=AffinityTopologyVersion [topVer=21, minorTopVer=0], exchId=null, partsMsg=null, exchangeNeeded=true], skipPreload=false, clientOnlyExchange=false, initTs=1487178539604, centralizedAff=false, evtLatch=0, remaining=[0940d968-a901-4c78-8354-8171628d41aa, 659ea029-ca80-4413-af47-67b446ffd3e7], srvNodes=[TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=0940d968-a901-4c78-8354-8171628d41aa, addrs=[0:0:0:0:0:0:0:1%lo, 10.129.1.129, 127.0.0.1], sockAddrs=[/10.129.1.129:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=20, intOrder=11, lastExchangeTime=1487178534516, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=79d74dfa-9903-42e3-af94-524ab08c1f97, addrs=[0:0:0:0:0:0:0:1%lo, 10.130.1.43, 127.0.0.1], sockAddrs=[webapp-92-50ik3/10.130.1.43:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=21, intOrder=12, lastExchangeTime=1487193806549, loc=true, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false]], super=GridFutureAdapter [resFlag=2, res=AffinityTopologyVersion [topVer=21, minorTopVer=1], startTime=1487178539604, endTime=1487178540040, ignoreInterrupts=false, state=DONE]] - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287670 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] >>> GridDhtPartitionsExchangeFuture [dummy=false, forcePreload=false, reassign=false, discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=79d74dfa-9903-42e3-af94-524ab08c1f97, addrs=[0:0:0:0:0:0:0:1%lo, 10.130.1.43, 127.0.0.1], sockAddrs=[webapp-92-50ik3/10.130.1.43:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=21, intOrder=12, lastExchangeTime=1487193806549, loc=true, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], topVer=21, nodeId8=79d74dfa, msg=null, type=NODE_JOINED, tstamp=1487178534607], crd=TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=21, minorTopVer=0], nodeId=79d74dfa, evt=NODE_JOINED], added=false, initFut=GridFutureAdapter [resFlag=2, res=true, startTime=1487178535485, endTime=1487178535753, ignoreInterrupts=false, state=DONE], init=true, topSnapshot=null, lastVer=null, partReleaseFut=GridCompoundFuture [rdc=null, initFlag=1, lsnrCalls=4, done=true, cancelled=false, err=null, futs=[true, true, true, true]], affChangeMsg=null, skipPreload=false, clientOnlyExchange=false, initTs=1487178535485, centralizedAff=false, evtLatch=0, remaining=[0940d968-a901-4c78-8354-8171628d41aa, 659ea029-ca80-4413-af47-67b446ffd3e7], srvNodes=[TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=0940d968-a901-4c78-8354-8171628d41aa, addrs=[0:0:0:0:0:0:0:1%lo, 10.129.1.129, 127.0.0.1], sockAddrs=[/10.129.1.129:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=20, intOrder=11, lastExchangeTime=1487178534516, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], TcpDiscoveryNode [id=79d74dfa-9903-42e3-af94-524ab08c1f97, addrs=[0:0:0:0:0:0:0:1%lo, 10.130.1.43, 127.0.0.1], sockAddrs=[webapp-92-50ik3/10.130.1.43:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=21, intOrder=12, lastExchangeTime=1487193806549, loc=true, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false]], super=GridFutureAdapter [resFlag=2, res=AffinityTopologyVersion [topVer=21, minorTopVer=0], startTime=1487178535485, endTime=1487178536184, ignoreInterrupts=false, state=DONE]] - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287670 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Pending transactions: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287670 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Pending explicit locks: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287670 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Pending cache futures: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287670 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Pending atomic cache futures: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287671 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] >>> GridDhtAtomicSingleUpdateFuture [key=KeyCacheObjectImpl [val=97B561DF-9EE8-6605-171B-B2404DDABCDA, hasValBytes=true], nearReaderEntry=null] - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287671 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] >>> GridDhtAtomicSingleUpdateFuture [key=KeyCacheObjectImpl [val=97B561DF-9EE8-6605-171B-B2404DDABCDA, hasValBytes=true], nearReaderEntry=null] - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287671 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Pending data streamer futures: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287671 WARN o.a.i.i.p.c.GridCachePartitionExchangeManager [exchange-worker-#26%webapp%] Pending transaction deadlock detection futures: - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287675 WARN o.a.i.s.c.tcp.TcpCommunicationSpi [exchange-worker-#26%webapp%] Communication SPI recovery descriptors: - Feb 16 08:23:28 webapp-92-50ik3 webapp [key=ClientKey [nodeId=0940d968-a901-4c78-8354-8171628d41aa, order=20], msgsSent=308295, msgsAckedByRmt=308288, msgsRcvd=286268, lastAcked=286267, reserveCnt=1, descIdHash=791992180] - Feb 16 08:23:28 webapp-92-50ik3 webapp [key=ClientKey [nodeId=659ea029-ca80-4413-af47-67b446ffd3e7, order=18], msgsSent=329991, msgsAckedByRmt=329984, msgsRcvd=346386, lastAcked=346385, reserveCnt=1, descIdHash=2140218415] - Feb 16 08:23:28 webapp-92-50ik3 webapp Communication SPI clients: - Feb 16 08:23:28 webapp-92-50ik3 webapp [node=659ea029-ca80-4413-af47-67b446ffd3e7, client=GridTcpNioCommunicationClient [ses=GridSelectorNioSessionImpl [selectorIdx=0, queueSize=0, writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], recovery=GridNioRecoveryDescriptor [acked=329984, resendCnt=0, rcvCnt=346386, sentCnt=329991, reserved=true, lastAck=346385, nodeLeft=false, node=TcpDiscoveryNode [id=659ea029-ca80-4413-af47-67b446ffd3e7, addrs=[0:0:0:0:0:0:0:1%lo, 10.128.1.11, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:47500, / 127.0.0.1:47500, /10.128.1.11:47500], discPort=47500, order=18, intOrder=10, lastExchangeTime=1487178534506, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], connected=true, connectCnt=0, queueLimit=5120, reserveCnt=1], super=GridNioSessionImpl [locAddr=/10.130.1.43:47100, rmtAddr=/10.128.1.11:51314, createTime=1487178534627, closeTime=0, bytesSent=344557012, bytesRcvd=620151134, sndSchedTime=1487193802122, lastSndTime=1487193803605, lastRcvTime=1487193806891, readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter [parser=o.a.i.i.util.nio.GridDirectParser@69c438c, directMode=true], GridConnectionBytesVerifyFilter], accepted=true]], super=GridAbstractCommunicationClient [lastUsed=1487178534637, reserves=0]]] - Feb 16 08:23:28 webapp-92-50ik3 webapp [node=0940d968-a901-4c78-8354-8171628d41aa, client=GridTcpNioCommunicationClient [ses=GridSelectorNioSessionImpl [selectorIdx=1, queueSize=0, writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], recovery=GridNioRecoveryDescriptor [acked=308288, resendCnt=0, rcvCnt=286268, sentCnt=308295, reserved=true, lastAck=286267, nodeLeft=false, node=TcpDiscoveryNode [id=0940d968-a901-4c78-8354-8171628d41aa, addrs=[0:0:0:0:0:0:0:1%lo, 10.129.1.129, 127.0.0.1], sockAddrs=[/10.129.1.129:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=20, intOrder=11, lastExchangeTime=1487178534516, loc=false, ver=1.8.1#20161208-sha1:a8fcb7b2, isClient=false], connected=true, connectCnt=0, queueLimit=5120, reserveCnt=1], super=GridNioSessionImpl [locAddr=/10.130.1.43:47100, rmtAddr=/10.129.1.129:53754, createTime=1487178536131, closeTime=0, bytesSent=325615325, bytesRcvd=362872151, sndSchedTime=1487193805754, lastSndTime=1487193805754, lastRcvTime=1487193807968, readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter [parser=o.a.i.i.util.nio.GridDirectParser@69c438c, directMode=true], GridConnectionBytesVerifyFilter], accepted=true]], super=GridAbstractCommunicationClient [lastUsed=1487178536131, reserves=0]]] - Feb 16 08:23:28 webapp-92-50ik3 webapp - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287676 WARN o.a.i.s.c.tcp.TcpCommunicationSpi [grid-nio-worker-2-#12%webapp%] - Feb 16 08:23:28 webapp-92-50ik3 webapp >> Selector info [idx=2, keysCnt=0] - Feb 16 08:23:28 webapp-92-50ik3 webapp - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287676 WARN o.a.i.s.c.tcp.TcpCommunicationSpi [grid-nio-worker-3-#13%webapp%] - Feb 16 08:23:28 webapp-92-50ik3 webapp >> Selector info [idx=3, keysCnt=0] - Feb 16 08:23:28 webapp-92-50ik3 webapp - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287678 WARN o.a.i.s.c.tcp.TcpCommunicationSpi [grid-nio-worker-0-#10%webapp%] - Feb 16 08:23:28 webapp-92-50ik3 webapp >> Selector info [idx=0, keysCnt=1] - Feb 16 08:23:28 webapp-92-50ik3 webapp Connection info [rmtAddr=/ 10.128.1.11:51314, locAddr=/10.130.1.43:47100, msgsSent=329991, msgsAckedByRmt=329984, msgsRcvd=346386, descIdHash=2140218415, bytesRcvd=620151134, bytesSent=344557012, opQueueSize=0, msgWriter=DirectMessageWriter [state=DirectMessageState [pos=0, stack=[StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396325296, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396325296, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396325296, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396325296, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396325296, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396325296, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], null, null, null, null]]], msgReader=DirectMessageReader [state=DirectMessageState [pos=0, stack=[StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396358080, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396358080, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396358080, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396358080, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713396358080, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], null, null, null, null, null]], lastRead=true]] - Feb 16 08:23:28 webapp-92-50ik3 webapp - Feb 16 08:23:28 webapp-92-50ik3 webapp 15287678 WARN o.a.i.s.c.tcp.TcpCommunicationSpi [grid-nio-worker-1-#11%webapp%] - Feb 16 08:23:28 webapp-92-50ik3 webapp >> Selector info [idx=1, keysCnt=1] - Feb 16 08:23:28 webapp-92-50ik3 webapp Connection info [rmtAddr=/ 10.129.1.129:53754, locAddr=/10.130.1.43:47100, msgsSent=308295, msgsAckedByRmt=308288, msgsRcvd=286268, descIdHash=791992180, bytesRcvd=362872151, bytesSent=325615325, opQueueSize=0, msgWriter=DirectMessageWriter [state=DirectMessageState [pos=0, stack=[StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463385408, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463385408, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463385408, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463385408, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463385408, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463385408, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0, hdrWritten=false], null, null, null, null]]], msgReader=DirectMessageReader [state=DirectMessageState [pos=0, stack=[StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463418192, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463418192, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463418192, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463418192, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2 [buf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], baseOff=140713463418192, arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], null, null, null, null, null]], lastRead=true]]
