[
https://issues.apache.org/jira/browse/IGNITE-3401?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15676116#comment-15676116
]
Ksenia Rybakova edited comment on IGNITE-3401 at 11/18/16 8:26 AM:
-------------------------------------------------------------------
Recently the issue is reproduced mostly during failover tests. And there are
just several errors during one test.
{noformat}
[01:09:52,065][INFO ][grid-timeout-worker-#7%null%][IgniteKernal]
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
^-- Node [id=91d112f5, name=null, uptime=00:19:36:667]
^-- H/N/C [hosts=8, nodes=25, CPUs=128]
^-- CPU [cur=3.77%, avg=5.76%, GC=0%]
^-- Heap [used=787MB, free=80.77%, comm=4096MB]
^-- Non heap [used=45MB, free=64.37%, comm=49MB]
^-- Public thread pool [active=0, idle=0, qSize=0]
^-- System thread pool [active=0, idle=32, qSize=0]
^-- Outbound messages queue [size=0]
[01:09:56,090][WARN ][grid-nio-worker-0-#9%null%][TcpCommunicationSpi] Failed
to process selector key (will close): GridSelectorNioSessionImpl
[selectorIdx=0, queueSize=0, writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768
cap=32768], re
adBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768],
recovery=GridNioRecoveryDescriptor [acked=163520, resendCnt=0, rcvCnt=163364,
sentCnt=163548, reserved=true, lastAck=163360, nodeLeft=false,
node=TcpDiscoveryNode [id=14eb506b-b
474-409f-84db-6391adc40582, addrs=[127.0.0.1, 172.25.1.35],
sockAddrs=[/172.25.1.35:47500, /127.0.0.1:47500], discPort=47500, order=27,
intOrder=26, lastExchangeTime=1479419999909, loc=false,
ver=1.7.0#20161115-sha1:6e36a795, isClient=fa
lse], connected=true, connectCnt=1, queueLimit=5120, reserveCnt=1],
super=GridNioSessionImpl [locAddr=/172.25.1.33:56562,
rmtAddr=/172.25.1.35:47100, createTime=1479420021901, closeTime=0,
bytesSent=251897508, bytesRcvd=25941223, sndSche
dTime=1479420596071, lastSndTime=1479420596071, lastRcvTime=1479420595864,
readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter
[parser=o.a.i.i.util.nio.GridDirectParser@3cb2db44, directMode=true],
GridConnectionBytesVe
rifyFilter], accepted=false]]
[01:09:56,091][WARN ][grid-nio-worker-0-#9%null%][TcpCommunicationSpi] Closing
NIO session because of unhandled exception [cls=class
o.a.i.i.util.nio.GridNioException, msg=Connection reset by peer]
[01:09:56,115][WARN ][tcp-comm-worker-#1%null%][TcpCommunicationSpi] Connect
timed out (consider increasing 'failureDetectionTimeout' configuration
property) [addr=/172.25.1.35:47100, failureDetectionTimeout=30000]
[01:09:56,116][WARN ][tcp-comm-worker-#1%null%][TcpCommunicationSpi] Failed to
connect to a remote node (make sure that destination node is alive and
operating system firewall is disabled on local and remote hosts)
[addrs=[/172.25.1.35:4
7100, /127.0.0.1:47100]]
[01:09:56,173][ERROR][benchmark-worker-29][GridDhtColocatedCache] <tx> Failed
to rollback transaction (cache may contain stale locks): GridNearTxLocal
[mappings=IgniteTxMappingsImpl [], nearLocallyMapped=false,
colocatedLocallyMapped=fal
se, needCheckBackup=null, hasRemoteLocks=false, thread=benchmark-worker-29,
mappings=IgniteTxMappingsImpl [], super=GridDhtTxLocalAdapter
[nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false,
super=IgniteTxLocalAda
pter [completedBase=null, sndTransformedVals=false, depEnabled=false,
txState=IgniteTxStateImpl [activeCacheIds=GridLongList [idx=1, arr=[3716]],
txMap={IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=889617446,
hash=2130805901, id=53083, code=id 53083], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=889617446,
hash=2130805901, id=53083, code=id 53083], cacheId=3716, partId=141,
txKey=IgniteTxKey [key=o.
a.i.yardstick.cache.load.model.key.Identifier [idHash=889617446,
hash=2130805901, id=53083, code=id 53083], cacheId=3716], val=[op=DELETE,
val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, val=null],
entryProcessorsCol=null, ttl=-
1, conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594981,
order=1479474333701, nodeOrder=20], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheE
ntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=889617446,
hash=2130805901, id=53083, code=id 53083], val=null, startVer=1479474331327,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331327,
nodeOrder=23], hash=2130805901, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=0f1dd901-2778-4f67-bb51-19e8d0356e05, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1019264508,
hash=2131177469, id=59245, code=id 59245], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1019264508,
hash=2131177469, id=59245, code=id 59245], cacheId=3716, partId=509,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1019264508, hash=2131177469, id=59245, code=id 59245], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595339, order=1479474338367, nodeOrder=14], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1019264508,
hash=2131177469, id=59245, code=id 59245], val=null, startVer=1479474331328,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331328,
nodeOrder=23], hash=2131177469, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=57fb05be-09bd-49a5-8497-270eaa219ac2, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=301533837,
hash=2131984494, id=62217, code=id 62217], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=301533837,
hash=2131984494, id=62217, code=id 62217], cacheId=3716, partId=110,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=301533837, hash=2131984494, id=62217, code=id 62217], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594935, order=1479474331745, nodeOrder=15], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=301533837,
hash=2131984494, id=62217, code=id 62217], val=null, startVer=1479474331329,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331329,
nodeOrder=23], hash=2131984494, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=634efa81-d094-4fad-8c59-351e1ab3f4bf, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1965275186,
hash=1466564606, id=111112, code=id 111112], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1965275186,
hash=1466564606, id=111112, code=id 111112], cacheId=3716, partId=510,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1965275186, hash=1466564606, id=111112, code=id 111112], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595714, order=1479474342037, nodeOrder=18], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1965275186,
hash=1466564606, id=111112, code=id 111112], val=null, startVer=1479474331330,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331330,
nodeOrder=23], hash=1466564606, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=67239319-f875-47c4-ac7a-4046e08f3236, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2081560242,
hash=1467994825, id=124475, code=id 124475], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2081560242,
hash=1467994825, id=124475, code=id 124475], cacheId=3716, partId=201,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=2081560242, hash=1467994825, id=124475, code=id 124475], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595692, order=1479474341667, nodeOrder=17], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2081560242,
hash=1467994825, id=124475, code=id 124475], val=null, startVer=1479474331331,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331331,
nodeOrder=23], hash=1467994825, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=464abd11-2258-4d57-95d5-f17d440a323a, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=126894017,
hash=1468319458, id=129986, code=id 129986], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=126894017,
hash=1468319458, id=129986, code=id 129986], cacheId=3716, partId=226,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=126894017, hash=1468319458, id=129986, code=id 129986], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594979, order=1479474333552, nodeOrder=27], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=126894017,
hash=1468319458, id=129986, code=id 129986], val=null, startVer=1479474331332,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331332,
nodeOrder=23], hash=1468319458, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=14eb506b-b474-409f-84db-6391adc40582, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=256482248,
hash=1471593120, id=152946, code=id 152946], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=256482248,
hash=1471593120, id=152946, code=id 152946], cacheId=3716, partId=160,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=256482248, hash=1471593120, id=152946, code=id 152946], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595766, order=1479474344803, nodeOrder=6], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=256482248,
hash=1471593120, id=152946, code=id 152946], val=null, startVer=1479474331333,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331333,
nodeOrder=23], hash=1471593120, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=21ca942d-9c8f-4324-8022-529e2b8f4004, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=697462607,
hash=1471650564, id=153867, code=id 153867], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=697462607,
hash=1471650564, id=153867, code=id 153867], cacheId=3716, partId=260,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=697462607, hash=1471650564, id=153867, code=id 153867], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594961, order=1479474333095, nodeOrder=10], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=697462607,
hash=1471650564, id=153867, code=id 153867], val=null, startVer=1479474331334,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331334,
nodeOrder=23], hash=1471650564, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=65136002-4551-4ca1-8e89-74746870c9a0, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1288290523,
hash=1474241949, id=175929, code=id 175929], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1288290523,
hash=1474241949, id=175929, code=id 175929], cacheId=3716, partId=413,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1288290523, hash=1474241949, id=175929, code=id 175929], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594981, order=1479474333701, nodeOrder=20], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1288290523,
hash=1474241949, id=175929, code=id 175929], val=null, startVer=1479474331335,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331335,
nodeOrder=23], hash=1474241949, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=0f1dd901-2778-4f67-bb51-19e8d0356e05, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1368691007,
hash=1476575311, id=193631, code=id 193631], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1368691007,
hash=1476575311, id=193631, code=id 193631], cacheId=3716, partId=79,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1368691007, hash=1476575311, id=193631, code=id 193631], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594981, order=1479474333701, nodeOrder=20], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1368691007,
hash=1476575311, id=193631, code=id 193631], val=null, startVer=1479474331336,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331336,
nodeOrder=23], hash=1476575311, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=0f1dd901-2778-4f67-bb51-19e8d0356e05, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1892411632,
hash=1499917754, id=227752, code=id 227752], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1892411632,
hash=1499917754, id=227752, code=id 227752], cacheId=3716, partId=442,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1892411632, hash=1499917754, id=227752, code=id 227752], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595766, order=1479474344803, nodeOrder=6], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1892411632,
hash=1499917754, id=227752, code=id 227752], val=null, startVer=1479474331337,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331337,
nodeOrder=23], hash=1499917754, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=21ca942d-9c8f-4324-8022-529e2b8f4004, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1529486954,
hash=1501030130, id=235765, code=id 235765], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1529486954,
hash=1501030130, id=235765, code=id 235765], cacheId=3716, partId=242,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1529486954, hash=1501030130, id=235765, code=id 235765], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596128, order=1479474350756, nodeOrder=3], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1529486954,
hash=1501030130, id=235765, code=id 235765], val=null, startVer=1479474331338,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331338,
nodeOrder=23], hash=1501030130, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=a1293739-255a-476f-b6de-d421c8cc6c29, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=707981288,
hash=1503254477, id=251779, code=id 251779], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=707981288,
hash=1503254477, id=251779, code=id 251779], cacheId=3716, partId=461,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=707981288, hash=1503254477, id=251779, code=id 251779], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596127, order=1479474350780, nodeOrder=8], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=707981288,
hash=1503254477, id=251779, code=id 251779], val=null, startVer=1479474331339,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331339,
nodeOrder=23], hash=1503254477, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=c68f9290-c806-430c-af6f-3af232157a8c, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1982103036,
hash=1503313563, id=252729, code=id 252729], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1982103036,
hash=1503313563, id=252729, code=id 252729], cacheId=3716, partId=155,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1982103036, hash=1503313563, id=252729, code=id 252729], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594979, order=1479474333552, nodeOrder=27], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1982103036,
hash=1503313563, id=252729, code=id 252729], val=null, startVer=1479474331340,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331340,
nodeOrder=23], hash=1503313563, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=14eb506b-b474-409f-84db-6391adc40582, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1439393788,
hash=1503320971, id=252908, code=id 252908], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1439393788,
hash=1503320971, id=252908, code=id 252908], cacheId=3716, partId=395,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1439393788, hash=1503320971, id=252908, code=id 252908], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594978, order=1479474333576, nodeOrder=9], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1439393788,
hash=1503320971, id=252908, code=id 252908], val=null, startVer=1479474331341,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331341,
nodeOrder=23], hash=1503320971, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=9690de93-2788-416b-a176-9afc19a99b22, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=488111461,
hash=1508479182, id=296445, code=id 296445], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=488111461,
hash=1508479182, id=296445, code=id 296445], cacheId=3716, partId=206,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=488111461, hash=1508479182, id=296445, code=id 296445], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595781, order=1479474345408, nodeOrder=16], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=488111461,
hash=1508479182, id=296445, code=id 296445], val=null, startVer=1479474331342,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331342,
nodeOrder=23], hash=1508479182, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=3947cd2f-cf30-40ed-856a-dace9c12042f, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=301793487,
hash=1531346798, id=322846, code=id 322846], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=301793487,
hash=1531346798, id=322846, code=id 322846], cacheId=3716, partId=366,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=301793487, hash=1531346798, id=322846, code=id 322846], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594981, order=1479474333701, nodeOrder=20], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=301793487,
hash=1531346798, id=322846, code=id 322846], val=null, startVer=1479474331343,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331343,
nodeOrder=23], hash=1531346798, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=0f1dd901-2778-4f67-bb51-19e8d0356e05, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2034245442,
hash=1531695481, id=328451, code=id 328451], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2034245442,
hash=1531695481, id=328451, code=id 328451], cacheId=3716, partId=377,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=2034245442, hash=1531695481, id=328451, code=id 328451], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596127, order=1479474350780, nodeOrder=8], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2034245442,
hash=1531695481, id=328451, code=id 328451], val=null, startVer=1479474331345,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331345,
nodeOrder=23], hash=1531695481, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=c68f9290-c806-430c-af6f-3af232157a8c, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1623483454,
hash=1534971317, id=351475, code=id 351475], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1623483454,
hash=1534971317, id=351475, code=id 351475], cacheId=3716, partId=437,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1623483454, hash=1534971317, id=351475, code=id 351475], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595819, order=1479474347183, nodeOrder=2], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1623483454,
hash=1534971317, id=351475, code=id 351475], val=null, startVer=1479474331347,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331347,
nodeOrder=23], hash=1534971317, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=3a7ce9ed-f037-4809-a0ab-105e22cc2550, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2106676243,
hash=1535330883, id=357342, code=id 357342], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2106676243,
hash=1535330883, id=357342, code=id 357342], cacheId=3716, partId=67,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=2106676243, hash=1535330883, id=357342, code=id 357342], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594981, order=1479474333701, nodeOrder=20], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2106676243,
hash=1535330883, id=357342, code=id 357342], val=null, startVer=1479474331351,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331351,
nodeOrder=23], hash=1535330883, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=0f1dd901-2778-4f67-bb51-19e8d0356e05, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1344728012,
hash=1537625411, id=374593, code=id 374593], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1344728012,
hash=1537625411, id=374593, code=id 374593], cacheId=3716, partId=323,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1344728012, hash=1537625411, id=374593, code=id 374593], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594981, order=1479474333711, nodeOrder=11], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1344728012,
hash=1537625411, id=374593, code=id 374593], val=null, startVer=1479474331357,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331357,
nodeOrder=23], hash=1537625411, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=c219e42d-5454-46b9-9389-df87305a3ec6, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=725796656,
hash=1539984387, id=392913, code=id 392913], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=725796656,
hash=1539984387, id=392913, code=id 392913], cacheId=3716, partId=3,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=725796656, hash=1539984387, id=392913, code=id 392913], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420594982, order=1479474333719, nodeOrder=19], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=725796656,
hash=1539984387, id=392913, code=id 392913], val=null, startVer=1479474331359,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331359,
nodeOrder=23], hash=1539984387, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=4d277cbc-ae08-4cf1-9c25-d0fe9becf567, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=757282432,
hash=1562203113, id=418746, code=id 418746], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=757282432,
hash=1562203113, id=418746, code=id 418746], cacheId=3716, partId=489,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=757282432, hash=1562203113, id=418746, code=id 418746], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595819, order=1479474347183, nodeOrder=2], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=757282432,
hash=1562203113, id=418746, code=id 418746], val=null, startVer=1479474331360,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331360,
nodeOrder=23], hash=1562203113, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=3a7ce9ed-f037-4809-a0ab-105e22cc2550, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1657742502,
hash=1563286902, id=426054, code=id 426054], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1657742502,
hash=1563286902, id=426054, code=id 426054], cacheId=3716, partId=374,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1657742502, hash=1563286902, id=426054, code=id 426054], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596127, order=1479474350780, nodeOrder=8], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1657742502,
hash=1563286902, id=426054, code=id 426054], val=null, startVer=1479474331361,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331361,
nodeOrder=23], hash=1563286902, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=c68f9290-c806-430c-af6f-3af232157a8c, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=225394936,
hash=1571574347, id=490492, code=id 490492], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=225394936,
hash=1571574347, id=490492, code=id 490492], cacheId=3716, partId=75,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=225394936, hash=1571574347, id=490492, code=id 490492], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596128, order=1479474350756, nodeOrder=3], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=225394936,
hash=1571574347, id=490492, code=id 490492], val=null, startVer=1479474331364,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331364,
nodeOrder=23], hash=1571574347, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=a1293739-255a-476f-b6de-d421c8cc6c29, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=51536044,
hash=1571683437, id=492183, code=id 492183], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=51536044,
hash=1571683437, id=492183, code=id 492183], cacheId=3716, partId=109,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=51536044, hash=1571683437, id=492183, code=id 492183], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596128, order=1479474350756, nodeOrder=3], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=51536044,
hash=1571683437, id=492183, code=id 492183], val=null, startVer=1479474331367,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331367,
nodeOrder=23], hash=1571683437, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=a1293739-255a-476f-b6de-d421c8cc6c29, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=730381767,
hash=1592574901, id=506699, code=id 506699], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=730381767,
hash=1592574901, id=506699, code=id 506699], cacheId=3716, partId=437,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=730381767, hash=1592574901, id=506699, code=id 506699], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595819, order=1479474347183, nodeOrder=2], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=730381767,
hash=1592574901, id=506699, code=id 506699], val=null, startVer=1479474331370,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331370,
nodeOrder=23], hash=1592574901, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=3a7ce9ed-f037-4809-a0ab-105e22cc2550, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=509831717,
hash=1602608633, id=589282, code=id 589282], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=509831717,
hash=1602608633, id=589282, code=id 589282], cacheId=3716, partId=505,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=509831717, hash=1602608633, id=589282, code=id 589282], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595339, order=1479474338367, nodeOrder=14], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=509831717,
hash=1602608633, id=589282, code=id 589282], val=null, startVer=1479474331373,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331373,
nodeOrder=23], hash=1602608633, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=57fb05be-09bd-49a5-8497-270eaa219ac2, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=281916089,
hash=1626810951, id=627180, code=id 627180], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=281916089,
hash=1626810951, id=627180, code=id 627180], cacheId=3716, partId=71,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=281916089, hash=1626810951, id=627180, code=id 627180], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595692, order=1479474341667, nodeOrder=17], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=281916089,
hash=1626810951, id=627180, code=id 627180], val=null, startVer=1479474331374,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331374,
nodeOrder=23], hash=1626810951, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=464abd11-2258-4d57-95d5-f17d440a323a, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=814782541,
hash=1630520233, id=657398, code=id 657398], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=814782541,
hash=1630520233, id=657398, code=id 657398], cacheId=3716, partId=425,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=814782541, hash=1630520233, id=657398, code=id 657398], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595819, order=1479474347183, nodeOrder=2], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=814782541,
hash=1630520233, id=657398, code=id 657398], val=null, startVer=1479474331376,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331376,
nodeOrder=23], hash=1630520233, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=3a7ce9ed-f037-4809-a0ab-105e22cc2550, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1694433315,
hash=1634235207, id=687744, code=id 687744], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1694433315,
hash=1634235207, id=687744, code=id 687744], cacheId=3716, partId=327,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1694433315, hash=1634235207, id=687744, code=id 687744], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595714, order=1479474342037, nodeOrder=18], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1694433315,
hash=1634235207, id=687744, code=id 687744], val=null, startVer=1479474331379,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331379,
nodeOrder=23], hash=1634235207, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=67239319-f875-47c4-ac7a-4046e08f3236, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=560082338,
hash=1659591037, id=734174, code=id 734174], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=560082338,
hash=1659591037, id=734174, code=id 734174], cacheId=3716, partId=381,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=560082338, hash=1659591037, id=734174, code=id 734174], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595744, order=1479474343660, nodeOrder=13], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=560082338,
hash=1659591037, id=734174, code=id 734174], val=null, startVer=1479474331384,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331384,
nodeOrder=23], hash=1659591037, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=0cbe08aa-4155-4486-8afa-e3ff13867015, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=883029965,
hash=1660784850, id=743697, code=id 743697], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=883029965,
hash=1660784850, id=743697, code=id 743697], cacheId=3716, partId=210,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=883029965, hash=1660784850, id=743697, code=id 743697], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595766, order=1479474344803, nodeOrder=6], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=883029965,
hash=1660784850, id=743697, code=id 743697], val=null, startVer=1479474331390,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331390,
nodeOrder=23], hash=1660784850, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=21ca942d-9c8f-4324-8022-529e2b8f4004, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1972620985,
hash=1663354342, id=765212, code=id 765212], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1972620985,
hash=1663354342, id=765212, code=id 765212], cacheId=3716, partId=486,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1972620985, hash=1663354342, id=765212, code=id 765212], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596127, order=1479474350780, nodeOrder=8], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1972620985,
hash=1663354342, id=765212, code=id 765212], val=null, startVer=1479474331395,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331395,
nodeOrder=23], hash=1663354342, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=c68f9290-c806-430c-af6f-3af232157a8c, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=988352509,
hash=1689121486, id=818798, code=id 818798], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=988352509,
hash=1689121486, id=818798, code=id 818798], cacheId=3716, partId=206,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=988352509, hash=1689121486, id=818798, code=id 818798], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595781, order=1479474345408, nodeOrder=16], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=988352509,
hash=1689121486, id=818798, code=id 818798], val=null, startVer=1479474331399,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331399,
nodeOrder=23], hash=1689121486, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=3947cd2f-cf30-40ed-856a-dace9c12042f, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=454609218,
hash=1725604789, id=955850, code=id 955850], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=454609218,
hash=1725604789, id=955850, code=id 955850], cacheId=3716, partId=437,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=454609218, hash=1725604789, id=955850, code=id 955850], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420595819, order=1479474347183, nodeOrder=2], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=454609218,
hash=1725604789, id=955850, code=id 955850], val=null, startVer=1479474331403,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331403,
nodeOrder=23], hash=1725604789, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=3a7ce9ed-f037-4809-a0ab-105e22cc2550, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2017794547,
hash=1725783783, id=958770, code=id 958770], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2017794547,
hash=1725783783, id=958770, code=id 958770], cacheId=3716, partId=231,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=2017794547, hash=1725783783, id=958770, code=id 958770], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596015, order=1479474348935, nodeOrder=1], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=2017794547,
hash=1725783783, id=958770, code=id 958770], val=null, startVer=1479474331410,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331410,
nodeOrder=23], hash=1725783783, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=aaabdc11-19b1-4404-b103-1bcc34d27c43, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=11198005,
hash=1728110708, id=976303, code=id 976303], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=11198005,
hash=1728110708, id=976303, code=id 976303], cacheId=3716, partId=116,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=11198005, hash=1728110708, id=976303, code=id 976303], cacheId=3716],
val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP,
val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1,
conflictVer=null, explicitVer=null, dhtVer=GridCacheVersion [topVer=90899423,
time=1479420596127, order=1479474350777, nodeOrder=12], filters=[],
filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=11198005,
hash=1728110708, id=976303, code=id 976303], val=null, startVer=1479474331415,
ver=GridCacheVersion [topVer=90899423, time=1479420594929, order=1479474331415,
nodeOrder=23], hash=1728110708, extras=null, flags=0]]], prepared=0,
locked=false, nodeId=b01f7c1d-4de8-4a29-9b96-6dfe2bb63d37, locMapped=false,
expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0,
serReadVer=null, xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1373792746,
hash=-1886375816, id=1002063, code=id 1002063], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1373792746,
hash=-1886375816, id=1002063, code=id 1002063], cacheId=3716, partId=392,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1373792746, hash=-1886375816, id=1002063, code=id 1002063],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420596127,
order=1479474350780, nodeOrder=8], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1373792746,
hash=-1886375816, id=1002063, code=id 1002063], val=null,
startVer=1479474331417, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331417, nodeOrder=23], hash=-1886375816,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=c68f9290-c806-430c-af6f-3af232157a8c, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1737993125,
hash=-1882670691, id=1032178, code=id 1032178], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1737993125,
hash=-1882670691, id=1032178, code=id 1032178], cacheId=3716, partId=99,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1737993125, hash=-1882670691, id=1032178, code=id 1032178],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420596128,
order=1479474350756, nodeOrder=3], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1737993125,
hash=-1882670691, id=1032178, code=id 1032178], val=null,
startVer=1479474331418, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331418, nodeOrder=23], hash=-1882670691,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=a1293739-255a-476f-b6de-d421c8cc6c29, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=245166785,
hash=-1880076247, id=1054310, code=id 1054310], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=245166785,
hash=-1880076247, id=1054310, code=id 1054310], cacheId=3716, partId=471,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=245166785, hash=-1880076247, id=1054310, code=id 1054310],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594979,
order=1479474333552, nodeOrder=27], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=245166785,
hash=-1880076247, id=1054310, code=id 1054310], val=null,
startVer=1479474331419, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331419, nodeOrder=23], hash=-1880076247,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=14eb506b-b474-409f-84db-6391adc40582, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=377885381,
hash=-1875393598, id=1090106, code=id 1090106], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=377885381,
hash=-1875393598, id=1090106, code=id 1090106], cacheId=3716, partId=62,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=377885381, hash=-1875393598, id=1090106, code=id 1090106],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420596127,
order=1479474350777, nodeOrder=12], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=377885381,
hash=-1875393598, id=1090106, code=id 1090106], val=null,
startVer=1479474331421, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331421, nodeOrder=23], hash=-1875393598,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=b01f7c1d-4de8-4a29-9b96-6dfe2bb63d37, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1010454288,
hash=-1851861193, id=1127411, code=id 1127411], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1010454288,
hash=-1851861193, id=1127411, code=id 1127411], cacheId=3716, partId=201,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1010454288, hash=-1851861193, id=1127411, code=id 1127411],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420595692,
order=1479474341667, nodeOrder=17], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1010454288,
hash=-1851861193, id=1127411, code=id 1127411], val=null,
startVer=1479474331422, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331422, nodeOrder=23], hash=-1851861193,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=464abd11-2258-4d57-95d5-f17d440a323a, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=404537940,
hash=-1849760006, id=1141398, code=id 1141398], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=404537940,
hash=-1849760006, id=1141398, code=id 1141398], cacheId=3716, partId=262,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=404537940, hash=-1849760006, id=1141398, code=id 1141398],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420596127,
order=1479474350780, nodeOrder=8], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=404537940,
hash=-1849760006, id=1141398, code=id 1141398], val=null,
startVer=1479474331426, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331426, nodeOrder=23], hash=-1849760006,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=c68f9290-c806-430c-af6f-3af232157a8c, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=135931178,
hash=-1845562424, id=1179654, code=id 1179654], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=135931178,
hash=-1845562424, id=1179654, code=id 1179654], cacheId=3716, partId=56,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=135931178, hash=-1845562424, id=1179654, code=id 1179654],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594981,
order=1479474333711, nodeOrder=11], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=135931178,
hash=-1845562424, id=1179654, code=id 1179654], val=null,
startVer=1479474331428, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331428, nodeOrder=23], hash=-1845562424,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=c219e42d-5454-46b9-9389-df87305a3ec6, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1820191885,
hash=-1811686121, id=1294263, code=id 1294263], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1820191885,
hash=-1811686121, id=1294263, code=id 1294263], cacheId=3716, partId=233,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1820191885, hash=-1811686121, id=1294263, code=id 1294263],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594961,
order=1479474333095, nodeOrder=10], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1820191885,
hash=-1811686121, id=1294263, code=id 1294263], val=null,
startVer=1479474331430, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331430, nodeOrder=23], hash=-1811686121,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=65136002-4551-4ca1-8e89-74746870c9a0, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1060459680,
hash=-1811425530, id=1298698, code=id 1298698], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1060459680,
hash=-1811425530, id=1298698, code=id 1298698], cacheId=3716, partId=250,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1060459680, hash=-1811425530, id=1298698, code=id 1298698],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594981,
order=1479474333701, nodeOrder=20], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1060459680,
hash=-1811425530, id=1298698, code=id 1298698], val=null,
startVer=1479474331432, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331432, nodeOrder=23], hash=-1811425530,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=0f1dd901-2778-4f67-bb51-19e8d0356e05, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1179055307,
hash=-1789877306, id=1313478, code=id 1313478], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1179055307,
hash=-1789877306, id=1313478, code=id 1313478], cacheId=3716, partId=58,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1179055307, hash=-1789877306, id=1313478, code=id 1313478],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594978,
order=1479474333576, nodeOrder=9], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1179055307,
hash=-1789877306, id=1313478, code=id 1313478], val=null,
startVer=1479474331435, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331435, nodeOrder=23], hash=-1789877306,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=9690de93-2788-416b-a176-9afc19a99b22, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1550933880,
hash=-1788697168, id=1322657, code=id 1322657], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1550933880,
hash=-1788697168, id=1322657, code=id 1322657], cacheId=3716, partId=80,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=1550933880, hash=-1788697168, id=1322657, code=id 1322657],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594981,
order=1479474333701, nodeOrder=20], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=1550933880,
hash=-1788697168, id=1322657, code=id 1322657], val=null,
startVer=1479474331437, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331437, nodeOrder=23], hash=-1788697168,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=0f1dd901-2778-4f67-bb51-19e8d0356e05, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]], IgniteTxKey
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=389213874,
hash=-1780083945, id=1392124, code=id 1392124], cacheId=3716]=IgniteTxEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=389213874,
hash=-1780083945, id=1392124, code=id 1392124], cacheId=3716, partId=233,
txKey=IgniteTxKey [key=o.a.i.yardstick.cache.load.model.key.Identifier
[idHash=389213874, hash=-1780083945, id=1392124, code=id 1392124],
cacheId=3716], val=[op=DELETE, val=null], prevVal=[op=NOOP, val=null],
oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1,
conflictExpireTime=-1, conflictVer=null, explicitVer=null,
dhtVer=GridCacheVersion [topVer=90899423, time=1479420594961,
order=1479474333095, nodeOrder=10], filters=[], filtersPassed=false,
filtersSet=true, entry=GridDhtDetachedCacheEntry
[super=GridDistributedCacheEntry [super=GridCacheMapEntry
[key=o.a.i.yardstick.cache.load.model.key.Identifier [idHash=389213874,
hash=-1780083945, id=1392124, code=id 1392124], val=null,
startVer=1479474331438, ver=GridCacheVersion [topVer=90899423,
time=1479420594929, order=1479474331438, nodeOrder=23], hash=-1780083945,
extras=null, flags=0]]], prepared=0, locked=false,
nodeId=65136002-4551-4ca1-8e89-74746870c9a0, locMapped=false, expiryPlc=null,
transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null,
xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23]]}], super=IgniteTxAdapter
[xidVer=GridCacheVersion [topVer=90899423, time=1479420594928,
order=1479474331275, nodeOrder=23], writeVer=null, implicit=true, loc=true,
threadId=241, startTime=1479420594919,
nodeId=91d112f5-c7d2-4fcc-905c-8a6ddd25513c, startVer=GridCacheVersion
[topVer=90899423, time=1479420594928, order=1479474331275, nodeOrder=23],
endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0,
sysInvalidate=false, sys=false, plc=2, commitVer=GridCacheVersion
[topVer=90899423, time=1479420594928, order=1479474331275, nodeOrder=23],
finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK,
timedOut=false, topVer=AffinityTopologyVersion [topVer=27, minorTopVer=1],
duration=1243ms, onePhaseCommit=false], size=50]]]
class org.apache.ignite.IgniteCheckedException: Failed to commit transaction:
GridNearTxLocal[id=b8eba877851-056b03df-0000-0017-0000-01587456caf0,
concurrency=OPTIMISTIC, isolation=READ_COMMITTED, state=ROLLED_BACK,
invalidate=false, rollbackOnly=true,
nodeId=91d112f5-c7d2-4fcc-905c-8a6ddd25513c, duration=1243]
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:433)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackAsync(GridNearTxLocal.java:928)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$14.applyx(IgniteTxLocalAdapter.java:3491)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$14.applyx(IgniteTxLocalAdapter.java:3481)
at
org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38)
at
org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:54)
at
org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:28)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:264)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:252)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:382)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:348)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:353)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:69)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:325)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:241)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:137)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:41)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:264)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:252)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:382)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:348)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:325)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$FinishMiniFuture.onNearFinishResponse(GridNearTxFinishFuture.java:1004)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:222)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishResponse(IgniteTxHandler.java:575)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$100(IgniteTxHandler.java:96)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$4.apply(IgniteTxHandler.java:158)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$4.apply(IgniteTxHandler.java:156)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:756)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:361)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:285)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$000(GridCacheIoManager.java:87)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:230)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1080)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:708)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$1700(GridIoManager.java:101)
at
org.apache.ignite.internal.managers.communication.GridIoManager$5.run(GridIoManager.java:671)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[01:09:56,247][ERROR][sys-#171%null%][IgniteTxManager] Failed to send one phase
commit ack to backup node [backup=14eb506b-b474-409f-84db-6391adc40582]
class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Node
left grid while sending message to: 14eb506b-b474-409f-84db-6391adc40582
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:834)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:977)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$2.finish(IgniteTxManager.java:283)
at
org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender$DeferredAckMessageBuffer.finish0(GridDeferredAckMessageSender.java:214)
at
org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender$DeferredAckMessageBuffer.access$000(GridDeferredAckMessageSender.java:111)
at
org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender$DeferredAckMessageBuffer$1.run(GridDeferredAckMessageSender.java:159)
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6598)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$1.body(GridClosureProcessor.java:765)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: class org.apache.ignite.IgniteCheckedException: Failed to send
message (node may have left the grid or TCP connection cannot be established
due to firewall issues) [node=TcpDiscoveryNode
[id=14eb506b-b474-409f-84db-6391adc40582, addrs=[127.0.0.1, 172.25.1.35],
sockAddrs=[/172.25.1.35:47500, /127.0.0.1:47500], discPort=47500, order=27,
intOrder=26, lastExchangeTime=1479419999909, loc=false,
ver=1.7.0#20161115-sha1:6e36a795, isClient=false], topic=TOPIC_CACHE,
msg=GridDhtTxOnePhaseCommitAckRequest [vers=[GridCacheVersion [topVer=90899423,
time=1479420595694, order=1479474340607, nodeOrder=23]], super=GridCacheMessage
[msgId=5747512, depInfo=null, err=null, skipPrepare=false, cacheId=0,
cacheId=0]], policy=2]
at
org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1151)
at
org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1215)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:828)
... 11 more
Caused by: class org.apache.ignite.spi.IgniteSpiException: Failed to send
message to remote node: TcpDiscoveryNode
[id=14eb506b-b474-409f-84db-6391adc40582, addrs=[127.0.0.1, 172.25.1.35],
sockAddrs=[/172.25.1.35:47500, /127.0.0.1:47500], discPort=47500, order=27,
intOrder=26, lastExchangeTime=1479419999909, loc=false,
ver=1.7.0#20161115-sha1:6e36a795, isClient=false]
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2017)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:1955)
at
org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1146)
... 13 more
Caused by: class org.apache.ignite.IgniteCheckedException: Failed to connect to
node (is node still alive?). Make sure that each ComputeTask and
GridCacheTransaction has a timeout set in order to prevent parties from waiting
forever in case of network issues [nodeId=14eb506b-b474-409f-84db-6391adc40582,
addrs=[/172.25.1.35:47100, /127.0.0.1:47100]]
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2521)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioClient(TcpCommunicationSpi.java:2161)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2055)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:1989)
... 15 more
Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to
connect to address: /172.25.1.35:47100
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2526)
... 18 more
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
at sun.nio.ch.SocketAdaptor.connect(SocketAdaptor.java:117)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2386)
... 18 more
Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to
connect to address: /127.0.0.1:47100
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2526)
... 18 more
Caused by: class org.apache.ignite.IgniteCheckedException: Remote node
ID is not as expected [expected=14eb506b-b474-409f-84db-6391adc40582,
rcvd=a1293739-255a-476f-b6de-d421c8cc6c29]
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeHandshake(TcpCommunicationSpi.java:2638)
... 18 more
[01:09:56,806][WARN ][disco-event-worker-#20%null%][GridDiscoveryManager] Node
FAILED: TcpDiscoveryNode [id=14eb506b-b474-409f-84db-6391adc40582,
addrs=[127.0.0.1, 172.25.1.35], sockAddrs=[/172.25.1.35:47500,
/127.0.0.1:47500], discPort=47500, order=27, intOrder=26,
lastExchangeTime=1479419999909, loc=false, ver=1.7.0#20161115-sha1:6e36a795,
isClient=false]
[01:09:56,807][INFO ][disco-event-worker-#20%null%][GridDiscoveryManager]
Topology snapshot [ver=28, servers=19, clients=5, CPUs=128, heap=170.0GB]
{noformat}
was (Author: krybakova):
Recently the issue is reproduced mostly during failover tests. And there are
just several errors during one test.
{noformat}
[01:09:52,065][INFO ][grid-timeout-worker-#7%null%][IgniteKernal]
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
^-- Node [id=91d112f5, name=null, uptime=00:19:36:667]
^-- H/N/C [hosts=8, nodes=25, CPUs=128]
^-- CPU [cur=3.77%, avg=5.76%, GC=0%]
^-- Heap [used=787MB, free=80.77%, comm=4096MB]
^-- Non heap [used=45MB, free=64.37%, comm=49MB]
^-- Public thread pool [active=0, idle=0, qSize=0]
^-- System thread pool [active=0, idle=32, qSize=0]
^-- Outbound messages queue [size=0]
[01:09:56,090][WARN ][grid-nio-worker-0-#9%null%][TcpCommunicationSpi] Failed
to process selector key (will close): GridSelectorNioSessionImpl
[selectorIdx=0, queueSize=0, writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768
cap=32768], re
[01:09:56,091][WARN ][grid-nio-worker-0-#9%null%][TcpCommunicationSpi] Closing
NIO session because of unhandled exception [cls=class
o.a.i.i.util.nio.GridNioException, msg=Connection reset by peer]
[01:09:56,115][WARN ][tcp-comm-worker-#1%null%][TcpCommunicationSpi] Connect
timed out (consider increasing 'failureDetectionTimeout' configuration
property) [addr=/172.25.1.35:47100, failureDetectionTimeout=30000]
[01:09:56,116][WARN ][tcp-comm-worker-#1%null%][TcpCommunicationSpi] Failed to
connect to a remote node (make sure that destination node is alive and
operating system firewall is disabled on local and remote hosts)
[addrs=[/172.25.1.35:4
[01:09:56,173][ERROR][benchmark-worker-29][GridDhtColocatedCache] <tx> Failed
to rollback transaction (cache may contain stale locks): GridNearTxLocal
[mappings=IgniteTxMappingsImpl [], nearLocallyMapped=false,
colocatedLocallyMapped=fal
class org.apache.ignite.IgniteCheckedException: Failed to commit transaction:
GridNearTxLocal[id=b8eba877851-056b03df-0000-0017-0000-01587456caf0,
concurrency=OPTIMISTIC, isolation=READ_COMMITTED, state=ROLLED_BACK,
invalidate=false, rol
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:433)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackAsync(GridNearTxLocal.java:928)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$14.applyx(IgniteTxLocalAdapter.java:3491)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter$14.applyx(IgniteTxLocalAdapter.java:3481)
at
org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38)
at
org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:54)
at
org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:28)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:264)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:252)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:382)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:348)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:353)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:69)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:325)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:241)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:137)
at
org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:41)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:264)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:252)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:382)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:348)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:325)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$FinishMiniFuture.onNearFinishResponse(GridNearTxFinishFuture.java:1004)
at
org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:222)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxFinishResponse(IgniteTxHandler.java:575)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$100(IgniteTxHandler.java:96)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$4.apply(IgniteTxHandler.java:158)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$4.apply(IgniteTxHandler.java:156)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:756)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:361)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:285)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$000(GridCacheIoManager.java:87)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:230)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1080)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:708)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$1700(GridIoManager.java:101)
at
org.apache.ignite.internal.managers.communication.GridIoManager$5.run(GridIoManager.java:671)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[01:09:56,247][ERROR][sys-#171%null%][IgniteTxManager] Failed to send one phase
commit ack to backup node [backup=14eb506b-b474-409f-84db-6391adc40582]
class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Node
left grid while sending message to: 14eb506b-b474-409f-84db-6391adc40582
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:834)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:977)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager$2.finish(IgniteTxManager.java:283)
at
org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender$DeferredAckMessageBuffer.finish0(GridDeferredAckMessageSender.java:214)
at
org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender$DeferredAckMessageBuffer.access$000(GridDeferredAckMessageSender.java:111)
at
org.apache.ignite.internal.processors.cache.GridDeferredAckMessageSender$DeferredAckMessageBuffer$1.run(GridDeferredAckMessageSender.java:159)
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6598)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$1.body(GridClosureProcessor.java:765)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: class org.apache.ignite.IgniteCheckedException: Failed to send
message (node may have left the grid or TCP connection cannot be established
due to firewall issues) [node=TcpDiscoveryNode
[id=14eb506b-b474-409f-84db-6391adc4058
at
org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1151)
at
org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1215)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:828)
... 11 more
Caused by: class org.apache.ignite.spi.IgniteSpiException: Failed to send
message to remote node: TcpDiscoveryNode
[id=14eb506b-b474-409f-84db-6391adc40582, addrs=[127.0.0.1, 172.25.1.35],
sockAddrs=[/172.25.1.35:47500, /127.0.0.1:47500]
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2017)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:1955)
at
org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1146)
... 13 more
Caused by: class org.apache.ignite.IgniteCheckedException: Failed to connect to
node (is node still alive?). Make sure that each ComputeTask and
GridCacheTransaction has a timeout set in order to prevent parties from waiting
forever in c
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2521)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioClient(TcpCommunicationSpi.java:2161)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2055)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:1989)
... 15 more
Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to
connect to address: /172.25.1.35:47100
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2526)
... 18 more
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
at sun.nio.ch.SocketAdaptor.connect(SocketAdaptor.java:117)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2386)
... 18 more
Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to
connect to address: /127.0.0.1:47100
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2526)
... 18 more
Caused by: class org.apache.ignite.IgniteCheckedException: Remote node
ID is not as expected [expected=14eb506b-b474-409f-84db-6391adc40582,
rcvd=a1293739-255a-476f-b6de-d421c8cc6c29]
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeHandshake(TcpCommunicationSpi.java:2638)
at
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2398)
... 18 more
[01:09:56,806][WARN ][disco-event-worker-#20%null%][GridDiscoveryManager] Node
FAILED: TcpDiscoveryNode [id=14eb506b-b474-409f-84db-6391adc40582,
addrs=[127.0.0.1, 172.25.1.35], sockAddrs=[/172.25.1.35:47500,
/127.0.0.1:47500], discPort=
[01:09:56,807][INFO ][disco-event-worker-#20%null%][GridDiscoveryManager]
Topology snapshot [ver=28, servers=19, clients=5, CPUs=128, heap=170.0GB]
{noformat}
> "Remote node ID is not as expected" errors at driver during load test
> ---------------------------------------------------------------------
>
> Key: IGNITE-3401
> URL: https://issues.apache.org/jira/browse/IGNITE-3401
> Project: Ignite
> Issue Type: Bug
> Components: general
> Reporter: Ksenia Rybakova
> Fix For: 1.9
>
>
> A lot of errors at some drivers after several minutes after grid is up during
> load test:
> {noformat}
> [14:33:42] (err) Failed to execute compound future reducer: Compound future
> listener []class org.apache.ignite.IgniteCheckedException: Failed to send
> message (node may have left the grid or TCP connection cannot be established
> due to fir
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1129)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1193)
> at
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$Buffer.submit(DataStreamerImpl.java:1378)
> at
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$Buffer.update(DataStreamerImpl.java:1173)
> at
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.load0(DataStreamerImpl.java:756)
> at
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.addDataInternal(DataStreamerImpl.java:568)
> at
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.addData(DataStreamerImpl.java:604)
> at
> org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark$1.run(IgniteCacheRandomOperationBenchmark.java:383)
> Caused by: class org.apache.ignite.spi.IgniteSpiException: Failed to send
> message to remote node: TcpDiscoveryNode
> [id=11da20f5-87c5-4321-8163-ffef85f2d54c, addrs=[10.20.0.215, 127.0.0.1],
> sockAddrs=[/127.0.0.1:47503, fosters-215/10.20.0
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:1986)
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:1926)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1124)
> ... 7 more
> Caused by: class org.apache.ignite.IgniteCheckedException: Failed to connect
> to node (is node still alive?). Make sure that each GridComputeTask and
> GridCacheTransaction has a timeout set in order to prevent parties from
> waiting forever
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2489)
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioClient(TcpCommunicationSpi.java:2130)
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2024)
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:1960)
> ... 9 more
> Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to
> connect to address: fosters-215/10.20.0.215:47103
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2494)
> ... 12 more
> Caused by: class org.apache.ignite.IgniteCheckedException: Failed to
> read remote node recovery handshake (connection closed).
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeHandshake(TcpCommunicationSpi.java:2699)
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2361)
> ... 12 more
> Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to
> connect to address: /127.0.0.1:47103
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2494)
> ... 12 more
> Caused by: class org.apache.ignite.IgniteCheckedException: Remote
> node ID is not as expected [expected=11da20f5-87c5-4321-8163-ffef85f2d54c,
> rcvd=03bc7d54-4fec-481d-af24-6f5408b32fe2]
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeHandshake(TcpCommunicationSpi.java:2604)
> at
> org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:2361)
> ... 12 more
> {noformat}
> Tested version: 1.5.27
> Load config:
> - 7 hosts with 4 servers and 3 drivers each (28 servers and 21 drivers in
> total)
> - benchmark: IgniteCacheRandomOperationBenchmark
> - preload: 2M
> - key range: 4M
> - warmup: 30s
> - duration: 10h
> - operations: PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL
> PUT_IF_ABSENT REPLACE
> -backup count: 1
> - GC: G1
> - heap size: 8Gb for servers, 4Gb for drivers
> - caches:
> {noformat}
> <bean name="atomic"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="atomic"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="ATOMIC"/>
> <property name="swapEnabled" value="false"/>
> </bean>
> <bean name="atomic-evict"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="atomic-evict"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="ATOMIC"/>
> <property name="swapEnabled" value="false"/>
> <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*6L}"/>
> <property name="evictionPolicy">
> <bean
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
> <property name="maxSize" value="750000"/>
> </bean>
> </property>
> </bean>
> <bean name="atomic-offheap"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="atomic-offheap"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="ATOMIC"/>
> <property name="swapEnabled" value="false"/>
> <property name="memoryMode" value="OFFHEAP_TIERED"/>
> </bean>
> <bean name="atomic-offheap-values"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="atomic-offheap-values"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="ATOMIC"/>
> <property name="swapEnabled" value="false"/>
> <property name="memoryMode" value="OFFHEAP_VALUES"/>
> </bean>
> <bean name="tx"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="tx"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="TRANSACTIONAL"/>
> <property name="swapEnabled" value="false"/>
> </bean>
> <bean name="tx-evict"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="tx-evict"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="TRANSACTIONAL"/>
> <property name="swapEnabled" value="false"/>
> <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*6L}"/>
> <property name="evictionPolicy">
> <bean
> class="org.apache.ignite.cache.eviction.lru.LruEvictionPolicy">
> <property name="maxSize" value="750000"/>
> </bean>
> </property>
> </bean>
> <bean name="compute"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="compute"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="TRANSACTIONAL"/>
> <property name="swapEnabled" value="false"/>
> <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*6L}"/>
> <property name="evictionPolicy">
> <bean
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
> <property name="maxSize" value="750000"/>
> </bean>
> </property>
> </bean>
> <bean name="tx-offheap"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="tx-offheap"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="TRANSACTIONAL"/>
> <property name="swapEnabled" value="false"/>
> <property name="memoryMode" value="OFFHEAP_TIERED"/>
> </bean>
> <bean name="tx-offheap-values"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="tx-offheap-values"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="TRANSACTIONAL"/>
> <property name="swapEnabled" value="false"/>
> <property name="memoryMode" value="OFFHEAP_VALUES"/>
> </bean>
> <bean name="atomic-index"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="atomic-index"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="ATOMIC"/>
> <property name="swapEnabled" value="false"/>
> <property name="indexedTypes">
> <list>
> <value>java.lang.Integer</value>
> <value>org.apache.ignite.yardstick.cache.model.Person1</value>
> </list>
> </property>
> </bean>
> <bean name="query"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="query"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="ATOMIC"/>
> <property name="swapEnabled" value="false"/>
> <!-- Configure query entities -->
> <property name="queryEntities">
> <list>
> <bean class="org.apache.ignite.cache.QueryEntity">
> <property name="keyType" value="java.lang.Integer"/>
> <property name="valueType"
> value="org.apache.ignite.yardstick.cache.model.Person"/>
> <property name="fields">
> <map>
> <entry key="id" value="java.lang.Integer"/>
> <entry key="orgId" value="java.lang.Integer"/>
> <entry key="firstName" value="java.lang.String"/>
> <entry key="lastName" value="java.lang.String"/>
> <entry key="salary" value="java.lang.Double"/>
> </map>
> </property>
> <property name="indexes">
> <list>
> <bean class="org.apache.ignite.cache.QueryIndex">
> <constructor-arg value="id"/>
> </bean>
> <!-- Group index. -->
> <bean class="org.apache.ignite.cache.QueryIndex">
> <constructor-arg>
> <list>
> <value>id</value>
> <value>salary</value>
> </list>
> </constructor-arg>
> <constructor-arg value="SORTED"/>
> </bean>
> </list>
> </property>
> </bean>
> </list>
> </property>
> </bean>
> <bean name="orgCache"
> class="org.apache.ignite.configuration.CacheConfiguration">
> <property name="name" value="orgCache"/>
> <property name="cacheMode" value="PARTITIONED"/>
> <property name="atomicityMode" value="ATOMIC"/>
> <property name="swapEnabled" value="false"/>
> <!-- Configure query entities -->
> <property name="queryEntities">
> <list>
> <bean class="org.apache.ignite.cache.QueryEntity">
> <property name="keyType" value="java.lang.Integer"/>
> <property name="valueType"
> value="org.apache.ignite.yardstick.cache.model.Organization"/>
> <property name="fields">
> <map>
> <entry key="id" value="java.lang.Integer"/>
> <entry key="name" value="java.lang.String"/>
> </map>
> </property>
> <property name="indexes">
> <list>
> <bean class="org.apache.ignite.cache.QueryIndex">
> <constructor-arg value="id"/>
> </bean>
> <bean class="org.apache.ignite.cache.QueryIndex">
> <constructor-arg value="name"/>
> </bean>
> </list>
> </property>
> </bean>
> </list>
> </property>
> </bean>
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
