Hi,
I have started four ignite nodes and configured cache in distributed mode.
When I initiated thousands of requests to write the data on this
cache(write through enabled) , facing below error.
>From logs we can see this error is occurring while witting to oracle
database.(using cache write-through).
This error is not consistent.Node does stop for while after this error and
continues to pick up the next ignite tasks.
Please someone advise what does following log means.
2018-11-13 05:52:05,577 2377545 [core-1] INFO
c.q.a.a.s.AssetManagementService - Add asset request processing started,
requestId ADD_Ip_483, subscriptionId =262604, userId=547159
2018-11-13 05:52:06,647 2378615 [grid-timeout-worker-#39%springDataNode%]
WARN o.a.ignite.internal.util.typedef.G - >>> Possible starvation in
striped pool.
Thread name: sys-stripe-11-#12%springDataNode%
Queue: [Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE,
topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false,
msg=GridNearSingleGetResponse [futId=1542085977929, res=BinaryObjectImpl
[arr= true, ctx=false, start=0], topVer=null, err=null, flags=0]]], Message
closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8,
ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtTxPrepareResponse
[nearEvicted=null, futId=b67ac7b0761-93ebea72-bf4e-40d8-8a19-d3258be94ce9,
miniId=1, super=GridDistributedTxPrepareResponse [txState=null, part=-1,
err=null, super=GridDistributedBaseMessage [ver=GridCacheVersion
[topVer=153565953, order=1542089536997, nodeOrder=3], committedVers=null,
rolledbackVers=null, cnt=0, super=GridCacheIdMessage [cacheId=0]]]]]],
Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8,
ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearSingleGetRequest
[futId=1542085976178, key=BinaryObjectImpl [arr= true, ctx=false, start=0],
flags=1, topVer=AffinityTopologyVersion [topVer=7, minorTopVer=0],
subjId=9e8db7e7-48ba-4161-881b-ad4fcfc175a0, taskNameHash=0, createTtl=-1,
accessTtl=-1]]]]
Deadlock: false
Completed: 703
Thread [name="sys-stripe-11-#12%springDataNode%", id=41, state=RUNNABLE,
blockCnt=37, waitCnt=729]
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at oracle.net.ns.Packet.receive(Packet.java:311)
at oracle.net.ns.DataPacket.receive(DataPacket.java:105)
at
oracle.net.ns.NetInputStream.getNextPacket(NetInputStream.java:305)
at oracle.net.ns.NetInputStream.read(NetInputStream.java:249)
at oracle.net.ns.NetInputStream.read(NetInputStream.java:171)
at oracle.net.ns.NetInputStream.read(NetInputStream.java:89)
at
oracle.jdbc.driver.T4CSocketInputStreamWrapper.readNextPacket(T4CSocketInputStreamWrapper.java:123)
at
oracle.jdbc.driver.T4CSocketInputStreamWrapper.read(T4CSocketInputStreamWrapper.java:79)
at
oracle.jdbc.driver.T4CMAREngineStream.unmarshalUB1(T4CMAREngineStream.java:429)
at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:397)
at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:257)
at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:587)
at
oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:225)
at
oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:53)
at
oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:943)
at
oracle.jdbc.driver.OraclePreparedStatement.executeForRowsWithTimeout(OraclePreparedStatement.java:12029)
at
oracle.jdbc.driver.OraclePreparedStatement.executeBatch(OraclePreparedStatement.java:12140)
at
oracle.jdbc.driver.OracleStatementWrapper.executeBatch(OracleStatementWrapper.java:246)
at
com.zaxxer.hikari.pool.ProxyStatement.executeBatch(ProxyStatement.java:128)
at
com.zaxxer.hikari.pool.HikariProxyPreparedStatement.executeBatch(HikariProxyPreparedStatement.java)
at
com.qualys.agms.grid.cache.loader.AbstractDefaultCacheStore.writeAll(AbstractDefaultCacheStore.java:126)
at
o.a.i.i.processors.cache.store.GridCacheStoreManagerAdapter.putAll(GridCacheStoreManagerAdapter.java:641)
at
o.a.i.i.processors.cache.transactions.IgniteTxAdapter.batchStoreCommit(IgniteTxAdapter.java:1422)
at
o.a.i.i.processors.cache.transactions.IgniteTxLocalAdapter.userCommit(IgniteTxLocalAdapter.java:502)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxLocal.localFinish(GridNearTxLocal.java:3185)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxFinishFuture.doFinish(GridNearTxFinishFuture.java:467)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:417)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxLocal$19.apply(GridNearTxLocal.java:3341)
at
o.a.i.i.processors.cache.distributed.near.GridNearTxLocal$19.apply(GridNearTxLocal.java:3335)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.processors.cache.GridCacheCompoundFuture.onDone(GridCacheCompoundFuture.java:56)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.processors.cache.distributed.near.GridNearOptimisticSerializableTxPrepareFuture.onComplete(GridNearOptimisticSerializableTxPrepareFuture.java:275)
at
o.a.i.i.processors.cache.distributed.near.GridNearOptimisticSerializableTxPrepareFuture.onDone(GridNearOptimisticSerializableTxPrepareFuture.java:220)
at
o.a.i.i.processors.cache.distributed.near.GridNearOptimisticSerializableTxPrepareFuture.onDone(GridNearOptimisticSerializableTxPrepareFuture.java:69)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.processors.cache.distributed.near.GridNearOptimisticSerializableTxPrepareFuture$MiniFuture.onResult(GridNearOptimisticSerializableTxPrepareFuture.java:963)
at
o.a.i.i.processors.cache.distributed.near.GridNearOptimisticSerializableTxPrepareFuture$1.apply(GridNearOptimisticSerializableTxPrepareFuture.java:583)
at
o.a.i.i.processors.cache.distributed.near.GridNearOptimisticSerializableTxPrepareFuture$1.apply(GridNearOptimisticSerializableTxPrepareFuture.java:580)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78)
at
o.a.i.i.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70)
at
o.a.i.i.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.processors.cache.GridCacheCompoundFuture.onDone(GridCacheCompoundFuture.java:56)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onComplete(GridDhtTxPrepareFuture.java:984)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:787)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onDone(GridDhtTxPrepareFuture.java:104)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144)
at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at
o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:495)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:474)
at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture$MiniFuture.onResult(GridDhtTxPrepareFuture.java:1870)
at
o.a.i.i.processors.cache.distributed.dht.GridDhtTxPrepareFuture.onResult(GridDhtTxPrepareFuture.java:549)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.processDhtTxPrepareResponse(IgniteTxHandler.java:708)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$500(IgniteTxHandler.java:97)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:207)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$6.apply(IgniteTxHandler.java:205)
at
o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
at
o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
at
o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
at
o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
at
o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
at
o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
at
o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
at
o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
at java.lang.Thread.run(Thread.java:748)
Thanks,
Akash
