Detailed error messageļ¼
Established outgoing communication connection [locAddr=/10.122.64.129:40870,
rmtAddr=/10.122.64.128:48339]
[01:01:59,775][SEVERE][grid-nio-worker-tcp-comm-1-#48%SubScriptionCluster%][TcpCommunicationSpi]
Failed to process selector key [ses=GridSelectorNioSessionImpl
[worker=DirectNioClientWorker [super=AbstractNioClientWorker [idx=1,
bytesRcvd=1151627, bytesSent=2289496, bytesRcvd0=64, bytesSent0=537,
select=true, super=GridWorker [name=grid-nio-worker-tcp-comm-1,
igniteInstanceName=SubScriptionCluster, finished=false, hashCode=440471613,
interrupted=false,
runner=grid-nio-worker-tcp-comm-1-#48%SubScriptionCluster%]]],
writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768],
readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768],
inRecovery=GridNioRecoveryDescriptor [acked=2651, resendCnt=0, rcvCnt=2752,
sentCnt=2652, reserved=true, lastAck=2752, nodeLeft=false,
node=TcpDiscoveryNode [id=09e38b03-ea63-47bf-a27b-661a0f0ab8d8,
addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.16.99.27],
sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.16.99.27:0],
discPort=0, order=44, intOrder=26, lastExchangeTime=1581939704719,
loc=false, ver=2.6.0#20180710-sha1:669feacc, isClient=true], connected=true,
connectCnt=0, queueLimit=4096, reserveCnt=225, pairedConnections=false],
outRecovery=GridNioRecoveryDescriptor [acked=2651, resendCnt=0, rcvCnt=2752,
sentCnt=2652, reserved=true, lastAck=2752, nodeLeft=false,
node=TcpDiscoveryNode [id=09e38b03-ea63-47bf-a27b-661a0f0ab8d8,
addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.16.99.27],
sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.16.99.27:0],
discPort=0, order=44, intOrder=26, lastExchangeTime=1581939704719,
loc=false, ver=2.6.0#20180710-sha1:669feacc, isClient=true], connected=true,
connectCnt=0, queueLimit=4096, reserveCnt=225, pairedConnections=false],
super=GridNioSessionImpl [locAddr=/10.122.64.129:48339,
rmtAddr=/10.122.23.179:33052, createTime=1582183927690, closeTime=0,
bytesSent=33096, bytesRcvd=29254, bytesSent0=211, bytesRcvd0=0,
sndSchedTime=1582246566604, lastSndTime=1582246918763,
lastRcvTime=1582246566604, readsPaused=false,
filterChain=FilterChain[filters=[GridNioCodecFilter
[parser=o.a.i.i.util.nio.GridDirectParser@7419c93, directMode=true],
GridConnectionBytesVerifyFilter], accepted=true]]]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
at
org.apache.ignite.internal.util.nio.GridNioServer$DirectNioClientWorker.processRead(GridNioServer.java:1250)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.processSelectedKeysOptimized(GridNioServer.java:2339)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2110)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1764)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:748)
[01:01:59,776][WARNING][grid-nio-worker-tcp-comm-1-#48%SubScriptionCluster%][TcpCommunicationSpi]
Closing NIO session because of unhandled exception [cls=class
o.a.i.i.util.nio.GridNioException, msg=Connection reset by peer]
[01:02:14,897][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/172.16.99.27:47100,
failureDetectionTimeout=100000]
[01:02:14,898][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/0:0:0:0:0:0:0:1%lo:47100,
failureDetectionTimeout=100000]
[01:02:14,898][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/127.0.0.1:47100,
failureDetectionTimeout=100000]
[01:02:14,898][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Failed to connect to a remote node (make sure that destination node is alive
and operating system firewall is disabled on local and remote hosts)
[addrs=[/172.16.99.27:47100, /0:0:0:0:0:0:0:1%lo:47100, /127.0.0.1:47100]]
[01:02:14,899][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:02:14,917][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=19ms]
[01:02:14,985][WARNING][grid-timeout-worker-#45%SubScriptionCluster%][G] >>>
Possible starvation in striped pool.
Thread name: sys-stripe-3-#4%SubScriptionCluster%
Queue: []
Deadlock: false
Completed: 96
Thread [name="sys-stripe-3-#4%SubScriptionCluster%", id=19,
state=TIMED_WAITING, blockCnt=1, waitCnt=215]
at java.lang.Thread.sleep(Native Method)
at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:7629)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1896)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1840)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1822)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1240)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1177)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:890)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:85)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:430)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:376)
at
o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1080)
at
o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:652)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:583)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:464)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:505)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitAsync(GridDhtTxLocal.java:526)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:735)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:104)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture$MiniFuture.onResult(GridDhtTxPrepareFuture.java:1870)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onResult(GridDhtTxPrepareFuture.java:549)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.processDhtTxPrepareResponse(IgniteTxHandler.java:708)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$500(IgniteTxHandler.java:97)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:207)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:205)
at
o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
at
o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
at
o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
at
o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
at
o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
at
o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
at
o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
at
o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
at java.lang.Thread.run(Thread.java:748)
[01:02:16,011][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:02:16,012][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=4ms]
[01:02:26,883][INFO][grid-timeout-worker-#45%SubScriptionCluster%][IgniteKernal%SubScriptionCluster]
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
^-- Node [id=bb0d277b, name=SubScriptionCluster, uptime=25:20:25.180]
^-- H/N/C [hosts=5, nodes=5, CPUs=32]
^-- CPU [cur=0.87%, avg=0.93%, GC=0%]
^-- PageMemory [pages=59694]
^-- Heap [used=1111MB, free=81.9%, comm=6144MB]
^-- Non heap [used=101MB, free=86.42%, comm=103MB]
^-- Outbound messages queue [size=0]
^-- Public thread pool [active=0, idle=0, qSize=0]
^-- System thread pool [active=0, idle=6, qSize=0]
[01:02:44,991][WARNING][grid-timeout-worker-#45%SubScriptionCluster%][G] >>>
Possible starvation in striped pool.
Thread name: sys-stripe-3-#4%SubScriptionCluster%
Queue: []
Deadlock: false
Completed: 96
Thread [name="sys-stripe-3-#4%SubScriptionCluster%", id=19, state=WAITING,
blockCnt=1, waitCnt=218]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304)
at
o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:177)
at
o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:140)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2882)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2692)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2651)
at
o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:1643)
at
o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:1863)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1869)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1840)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1822)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1240)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1177)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:890)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:85)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:430)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:376)
at
o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1080)
at
o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:652)
at
o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:1643)
at
o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:1863)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1869)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1840)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1822)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1240)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1177)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:890)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:85)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:430)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:376)
at
o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1080)
at
o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:652)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:583)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:464)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:505)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitAsync(GridDhtTxLocal.java:526)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:735)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:104)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture$MiniFuture.onResult(GridDhtTxPrepareFuture.java:1870)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onResult(GridDhtTxPrepareFuture.java:549)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.processDhtTxPrepareResponse(IgniteTxHandler.java:708)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$500(IgniteTxHandler.java:97)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:207)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:205)
at
o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
at
o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
at
o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
at
o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
at
o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
at
o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
at
o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
at
o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
at java.lang.Thread.run(Thread.java:748)
[01:02:47,071][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:02:47,072][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=9ms]
[01:02:47,071][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:02:47,072][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=9ms]
[01:03:15,000][WARNING][grid-timeout-worker-#45%SubScriptionCluster%][G] >>>
Possible starvation in striped pool.
Thread name: sys-stripe-3-#4%SubScriptionCluster%
Queue: []
Deadlock: false
Completed: 96
Thread [name="sys-stripe-3-#4%SubScriptionCluster%", id=19, state=WAITING,
blockCnt=1, waitCnt=220]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304)
at
o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:177)
at
o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:140)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2882)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2692)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2651)
at
o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:1643)
at
o.a.i.i.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:1863)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1869)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1840)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1822)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1240)
at
o.a.i.i.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1177)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:890)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:85)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:430)
at
o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:376)
at
o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1080)
at
o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:652)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:583)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:464)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:505)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxLocal.commitAsync(GridDhtTxLocal.java:526)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:735)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:104)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture$MiniFuture.onResult(GridDhtTxPrepareFuture.java:1870)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onResult(GridDhtTxPrepareFuture.java:549)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.processDhtTxPrepareResponse(IgniteTxHandler.java:708)
59619,9
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture$MiniFuture.onResult(GridDhtTxPrepareFuture.java:1870)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onResult(GridDhtTxPrepareFuture.java:549)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.processDhtTxPrepareResponse(IgniteTxHandler.java:708)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$500(IgniteTxHandler.java:97)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:207)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:205)
at
o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
at
o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
at
o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
at
o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
at
o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
at
o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
at
o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
at
o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
at java.lang.Thread.run(Thread.java:748)
[01:03:26,885][INFO][grid-timeout-worker-#45%SubScriptionCluster%][IgniteKernal%SubScriptionCluster]
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
^-- Node [id=bb0d277b, name=SubScriptionCluster, uptime=25:21:25.186]
^-- H/N/C [hosts=5, nodes=5, CPUs=32]
^-- CPU [cur=0.9%, avg=0.93%, GC=0%]
^-- PageMemory [pages=59694]
^-- Heap [used=1137MB, free=81.48%, comm=6144MB]
^-- Non heap [used=101MB, free=86.42%, comm=103MB]
^-- Outbound messages queue [size=0]
^-- Public thread pool [active=0, idle=0, qSize=0]
^-- System thread pool [active=0, idle=6, qSize=0]
[01:04:34,595][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:04:34,597][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=5ms]
[01:05:37,760][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:05:37,761][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=8ms]
[01:05:38,855][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:05:38,856][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=2ms]
[01:05:45,888][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:05:45,889][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=4ms]
[01:07:25,966][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/172.16.99.27:47100,
failureDetectionTimeout=100000]
[01:07:25,967][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/0:0:0:0:0:0:0:1%lo:47100,
failureDetectionTimeout=100000]
[01:07:25,967][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/127.0.0.1:47100,
failureDetectionTimeout=100000]
[01:07:25,968][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Failed to connect to a remote node (make sure that destination node is alive
and operating system firewall is disabled on local and remote hosts)
[addrs=[/172.16.99.27:47100, /0:0:0:0:0:0:0:1%lo:47100, /127.0.0.1:47100]]
[01:07:27,270][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:07:27,272][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=7ms]
[01:07:28,299][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:07:28,300][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=4ms]
[01:07:28,321][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
59
[01:07:25,966][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/172.16.99.27:47100,
failureDetectionTimeout=100000]
[01:07:25,967][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/0:0:0:0:0:0:0:1%lo:47100,
failureDetectionTimeout=100000]
[01:07:25,967][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Connect timed out (consider increasing 'failureDetectionTimeout'
configuration property) [addr=/127.0.0.1:47100,
failureDetectionTimeout=100000]
[01:07:25,968][WARNING][tcp-comm-worker-#1%SubScriptionCluster%][TcpCommunicationSpi]
Failed to connect to a remote node (make sure that destination node is alive
and operating system firewall is disabled on local and remote hosts)
[addrs=[/172.16.99.27:47100, /0:0:0:0:0:0:0:1%lo:47100, /127.0.0.1:47100]]
[01:07:27,270][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:07:27,272][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=7ms]
[01:07:28,299][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:07:28,300][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=4ms]
[01:07:28,321][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:07:28,323][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Finished node ping [nodeId=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, res=true,
time=7ms]
[01:07:29,340][INFO][tcp-comm-worker-#1%SubScriptionCluster%][TcpDiscoverySpi]
Pinging node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
[01:07:29,340][SEVERE][sys-stripe-3-#4%SubScriptionCluster%][query] Failed
to send event notification to node: 09e38b03-ea63-47bf-a27b-661a0f0ab8d8
class org.apache.ignite.IgniteCheckedException: Failed to send message (node
may have left the grid or TCP connection cannot be established due to
firewall issues) [node=TcpDiscoveryNode
[id=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, addrs=[0:0:0:0:0:0:0:1%lo,
127.0.0.1, 172.16.99.27], sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0,
/172.16.99.27:0], discPort=0, order=44, intOrder=26,
lastExchangeTime=1581939704719, loc=false, ver=2.6.0#20180710-sha1:669feacc,
isClient=true], topic=T4 [topic=TOPIC_CACHE,
id1=651b3302-d586-38ee-89b9-965a111ee94e,
id2=09e38b03-ea63-47bf-a27b-661a0f0ab8d8, id3=0], msg=GridContinuousMessage
[type=MSG_EVT_NOTIFICATION, routineId=d807ddd1-7fdf-4792-9324-6fffa78ad24c,
data=null, futId=null], policy=2]
at
org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1651)
at
org.apache.ignite.internal.managers.communication.GridIoManager.sendOrderedMessage(GridIoManager.java:1863)
at
org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1869)
at
org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1840)
at
org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1822)
at
org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:1240)
at
org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:1177)
at
org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.onEntryUpdate(CacheContinuousQueryHandler.java:890)
at
org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler.access$600(CacheContinuousQueryHandler.java:85)
at
org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler$2.onEntryUpdated(CacheContinuousQueryHandler.java:430)
at
org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.onEntryUpdated(CacheContinuousQueryManager.java:376)
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1080)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:652)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.localFinish(GridDhtTxLocalAdapter.java:795)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.localFinish(GridDhtTxLocal.java:583)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.finishTx(GridDhtTxLocal.java:464)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitDhtLocalAsync(GridDhtTxLocal.java:505)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.commitAsync(GridDhtTxLocal.java:526)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:735)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:104)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
Then the node keeps reporting errors and the node cannot be used.
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
