Unfortunately iam unable to send the full logs files but they contain the
following exceptions
[14 Sep 2016 11:14:30.290 EDT] [pub-#16%DataGridServer-Development%] ERROR
11223 (OrderHolderSaveRunnable.java:273) exception ocurred while generating
Trade Order for Order: OrderKey [traderId=5
207, orderId=16084348]
javax.cache.CacheException: class
org.apache.ignite.transactions.TransactionTimeoutException: Failed to
acquire lock within provided timeout for transaction [timeout=5000,
tx=GridNearTxLocal [ma
ppings=IgniteTxMappingsImpl [], nearLocallyMapped=false,
colocatedLocallyMapped=false, needCheckBackup=null, hasRemoteLocks=false,
mappings=IgniteTxMappingsImpl [], super=GridDhtTxLocalAdapter [
nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false,
super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false,
depEnabled=false, txState=IgniteTxStateImpl
[activeCacheIds=GridLongList [idx=1, arr=[1633849959]], txMap={IgniteTxKey
[key=KeyCacheObjectImpl [val=BatchIdKey [privDb=trim_sys],
hasValBytes=true], cacheId=1633849959]=IgniteTxEntry [key=Ke
yCacheObjectImpl [val=BatchIdKey [privDb=trim_sys], hasValBytes=true],
cacheId=1633849959, txKey=IgniteTxKey [key=KeyCacheObjectImpl
[val=BatchIdKey [privDb=trim_sys], hasValBytes=true], cacheId
=1633849959], val=[op=READ, val=null], prevVal=[op=NOOP, val=null],
entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null,
explicitVer=null, dhtVer=null, filters=null, filters
Passed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=KeyCacheObjectImpl [val=BatchIdKey [privDb=trim_sys], hasValBytes=tr
ue], val=null, startVer=1473869129773, ver=GridCacheVersion
[topVer=85333522, nodeOrderDrId=10, globalTime=1473859812640,
order=1473869129773], hash=1508409679, extras=null, flags=0]]], prepared
=false, locked=false, nodeId=3cd37805-46a7-4287-875e-9cbd0cf03fad,
locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0,
partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [
topVer=85333522, nodeOrderDrId=10, globalTime=1473859812640,
order=1473869129772]]}], super=IgniteTxAdapter [xidVer=GridCacheVersion
[topVer=85333522, nodeOrderDrId=10, globalTime=1473859812640,
order=1473869129772], writeVer=null, implicit=false, loc=true, threadId=50,
startTime=1473859812630, nodeId=6f7a39ba-c520-435e-9480-a42ecf0d9a58,
startVer=GridCacheVersion [topVer=85333522, nod
eOrderDrId=10, globalTime=1473859812640, order=1473869129772], endVer=null,
isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=5000,
sysInvalidate=false, sys=false, plc=2, commitVer=nul
l, finalizing=NONE, preparing=false, invalidParts=null,
state=MARKED_ROLLBACK, timedOut=false, topVer=AffinityTopologyVersion
[topVer=101, minorTopVer=0], duration=5007ms, onePhaseCommit=false],
size=1]]]]
at
org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1618)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1841)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.cache.IgniteCacheProxy.get(IgniteCacheProxy.java:871)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
com.somecompany.grid.server.tradegen.BatchIdHelper.getListOfIds(BatchIdHelper.java:69)
~[data-grid-server-ignite-3.0-SNAPSHOT.jar:3.0-SNAPSHOT]
at
com.somecompany.grid.server.tradegen.TradeGenerator.generateUniqueTradeId64(TradeGenerator.java:47)
~[data-grid-server-ignite-3.0-SNAPSHOT.jar:3.0-SNAPSHOT]
at
com.somecompany.grid.server.tradegen.TradeGenerator.allocateTradesFromFills(TradeGenerator.java:158)
~[data-grid-server-ignite-3.0-SNAPSHOT.jar:3.0-SNAPSHOT]
at
com.somecompany.grid.server.tradegen.OrderHolderSaveRunnable.run(OrderHolderSaveRunnable.java:271)
~[data-grid-server-ignite-3.0-SNAPSHOT.jar:3.0-SNAPSHOT]
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$C4.execute(GridClosureProcessor.java:1879)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.job.GridJobWorker$2.call(GridJobWorker.java:509)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6397)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:503)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:456)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.job.GridJobProcessor.processJobExecuteRequest(GridJobProcessor.java:1166)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.job.GridJobProcessor$JobExecutionListener.onMessage(GridJobProcessor.java:1770)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:821)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$1600(GridIoManager.java:103)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.managers.communication.GridIoManager$5.run(GridIoManager.java:784)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0_60]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0_60]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
Caused by: org.apache.ignite.transactions.TransactionTimeoutException:
Failed to acquire lock within provided timeout for transaction
[timeout=5000, tx=GridNearTxLocal [mappings=IgniteTxMappings
Impl [], nearLocallyMapped=false, colocatedLocallyMapped=false,
needCheckBackup=null, hasRemoteLocks=false, mappings=IgniteTxMappingsImpl
[], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=f
alse, nearNodes=[], dhtNodes=[], explicitLock=false,
super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false,
depEnabled=false, txState=IgniteTxStateImpl [activeCacheIds=GridLon
gList [idx=1, arr=[1633849959]], txMap={IgniteTxKey [key=KeyCacheObjectImpl
[val=BatchIdKey [privDb=trim_sys], hasValBytes=true],
cacheId=1633849959]=IgniteTxEntry [key=KeyCacheObjectImpl [val=B
atchIdKey [privDb=trim_sys], hasValBytes=true], cacheId=1633849959,
txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BatchIdKey [privDb=trim_sys],
hasValBytes=true], cacheId=1633849959], val=[op=R
EAD, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=null,
ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=null, filters=null, filtersPassed=false, filtersSe
t=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry
[super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BatchIdKey
[privDb=trim_sys], hasValBytes=true], val=null, startVer
=1473869129773, ver=GridCacheVersion [topVer=85333522, nodeOrderDrId=10,
globalTime=1473859812640, order=1473869129773], hash=1508409679,
extras=null, flags=0]]], prepared=false, locked=false, n
odeId=3cd37805-46a7-4287-875e-9cbd0cf03fad, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=85333522, nodeOr
derDrId=10, globalTime=1473859812640, order=1473869129772]]}],
super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=85333522,
nodeOrderDrId=10, globalTime=1473859812640, order=1473869129772],
writeVer=null, implicit=false, loc=true, threadId=50,
startTime=1473859812630, nodeId=6f7a39ba-c520-435e-9480-a42ecf0d9a58,
startVer=GridCacheVersion [topVer=85333522, nodeOrderDrId=10, globalTi
me=1473859812640, order=1473869129772], endVer=null,
isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=5000,
sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, pre
paring=false, invalidParts=null, state=MARKED_ROLLBACK, timedOut=false,
topVer=AffinityTopologyVersion [topVer=101, minorTopVer=0], duration=5007ms,
onePhaseCommit=false], size=1]]]]
at
org.apache.ignite.internal.util.IgniteUtils$12.apply(IgniteUtils.java:791)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.IgniteUtils$12.apply(IgniteUtils.java:789)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
... 21 more
Caused by:
org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException:
Failed to acquire lock within provided timeout for transaction
[timeout=5000, tx=GridNearTxLocal [mappings=Ign
iteTxMappingsImpl [], nearLocallyMapped=false, colocatedLocallyMapped=false,
needCheckBackup=null, hasRemoteLocks=false, mappings=IgniteTxMappingsImpl
[], super=GridDhtTxLocalAdapter [nearOnOrig
inatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false,
super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false,
depEnabled=false, txState=IgniteTxStateImpl [activeCac
heIds=GridLongList [idx=1, arr=[1633849959]], txMap={IgniteTxKey
[key=KeyCacheObjectImpl [val=BatchIdKey [privDb=trim_sys],
hasValBytes=true], cacheId=1633849959]=IgniteTxEntry [key=KeyCacheObje
ctImpl [val=BatchIdKey [privDb=trim_sys], hasValBytes=true],
cacheId=1633849959, txKey=IgniteTxKey [key=KeyCacheObjectImpl
[val=BatchIdKey [privDb=trim_sys], hasValBytes=true], cacheId=163384995
9], val=[op=READ, val=null], prevVal=[op=NOOP, val=null],
entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null,
explicitVer=null, dhtVer=null, filters=null, filtersPassed=fal
se, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=KeyCacheObjectImpl [val=BatchIdKey [privDb=trim_sys],
hasValBytes=true], val=n
ull, startVer=1473869129773, ver=GridCacheVersion [topVer=85333522,
nodeOrderDrId=10, globalTime=1473859812640, order=1473869129773],
hash=1508409679, extras=null, flags=0]]], prepared=false, lo
cked=false, nodeId=3cd37805-46a7-4287-875e-9cbd0cf03fad, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=853
33522, nodeOrderDrId=10, globalTime=1473859812640, order=1473869129772]]}],
super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=85333522,
nodeOrderDrId=10, globalTime=1473859812640, order=147
3869129772], writeVer=null, implicit=false, loc=true, threadId=50,
startTime=1473859812630, nodeId=6f7a39ba-c520-435e-9480-a42ecf0d9a58,
startVer=GridCacheVersion [topVer=85333522, nodeOrderDrId
=10, globalTime=1473859812640, order=1473869129772], endVer=null,
isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=5000,
sysInvalidate=false, sys=false, plc=2, commitVer=null, finaliz
ing=NONE, preparing=false, invalidParts=null, state=MARKED_ROLLBACK,
timedOut=false, topVer=AffinityTopologyVersion [topVer=101, minorTopVer=0],
duration=5007ms, onePhaseCommit=false], size=1]]]
]
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$PostLockClosure2.apply(IgniteTxLocalAdapter.java:4023)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$PostLockClosure2.apply(IgniteTxLocalAdapter.java:4010)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.future.GridEmbeddedFuture$3.applyx(GridEmbeddedFuture.java:158)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.future.GridEmbeddedFuture$AsyncListener1.apply(GridEmbeddedFuture.java:297)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.future.GridEmbeddedFuture$AsyncListener1.apply(GridEmbeddedFuture.java:290)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:262)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:250)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:380)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:346)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.onComplete(GridDhtColocatedLockFuture.java:535)
~[ignite-core-1.5.0.final.jar:1.5.0.fi
nal]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture.access$1100(GridDhtColocatedLockFuture.java:78)
~[ignite-core-1.5.0.final.jar:1.5.0.fi
nal]
at
org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture$LockTimeoutObject.onTimeout(GridDhtColocatedLockFuture.java:1291)
~[ignite-core-1.5.0.
final.jar:1.5.0.final]
at
org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:159)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
~[ignite-core-1.5.0.final.jar:1.5.0.final]
--
View this message in context:
http://apache-ignite-users.70518.x6.nabble.com/One-failing-node-stalling-the-whole-cluster-tp5372p7809.html
Sent from the Apache Ignite Users mailing list archive at Nabble.com.
