[
https://issues.apache.org/jira/browse/IGNITE-6580?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Mikhail Cherkasov updated IGNITE-6580:
--------------------------------------
Description:
The following exceptions can be abserved during concurrent re-balancing and
cache destruction:
1.
{noformat}
[00:01:27,135][ERROR][sys-#4375%null%][GridDhtPreloader] Partition eviction
failed, this can cause grid hang.
org.apache.ignite.IgniteException: Runtime failure on search row: Row@6be51c3d[
**REMOVED SENSITIVE INFORMATION** ]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1787)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.remove(BPlusTree.java:1578)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex.remove(H2TreeIndex.java:226)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:523)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:416)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:574)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2172)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1462)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1425)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3224)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry.clearInternal(GridDhtCacheEntry.java:588)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.clearAll(GridDhtLocalPartition.java:951)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.tryEvict(GridDhtLocalPartition.java:809)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:593)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:580)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6629)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$2.body(GridClosureProcessor.java:967)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
[ignite-core-2.1.4.jar:2.1.4]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0_131]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0_131]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: java.lang.IllegalStateException: Item not found: 1
at
org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.findIndirectItemIndex(DataPageIO.java:346)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.getDataOffset(DataPageIO.java:446)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.readPayload(DataPageIO.java:488)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:149)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:101)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2RowFactory.getRow(H2RowFactory.java:62)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:60)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:33)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:123)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:40)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:4372)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:164)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:40)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.compare(BPlusTree.java:4359)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findInsertionPoint(BPlusTree.java:4279)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$1500(BPlusTree.java:81)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Search.run0(BPlusTree.java:261)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4697)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4682)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readPage(PageHandler.java:158)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.DataStructure.read(DataStructure.java:319)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1823)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1752)
~[ignite-core-2.1.4.jar:2.1.4]
... 22 more
{noformat}
2.
{noformat}
[00:01:27,454][ERROR][sys-#4375%null%][GridDhtPreloader] Partition eviction
failed, this can cause grid hang.
org.apache.ignite.IgniteException: Runtime failure on search row: Row@146ed070[
**REMOVED SENSITIVE INFORMATION**]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1787)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.remove(BPlusTree.java:1578)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex.remove(H2TreeIndex.java:226)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:523)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:416)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:574)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2172)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1462)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1425)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3224)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry.clearInternal(GridDhtCacheEntry.java:588)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.clearAll(GridDhtLocalPartition.java:951)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.tryEvict(GridDhtLocalPartition.java:809)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:593)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:580)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6629)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$2.body(GridClosureProcessor.java:967)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
[ignite-core-2.1.4.jar:2.1.4]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0_131]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0_131]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: org.apache.ignite.binary.BinaryObjectException: Failed to get field
because type ID of passed object differs from type ID this BinaryField belongs
to [expected=-1565418643, actual=-461073156]
at
org.apache.ignite.internal.binary.BinaryFieldImpl.fieldOrder(BinaryFieldImpl.java:274)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.binary.BinaryFieldImpl.value(BinaryFieldImpl.java:106)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.property.QueryBinaryProperty.fieldValue(QueryBinaryProperty.java:239)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.property.QueryBinaryProperty.value(QueryBinaryProperty.java:135)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.H2RowDescriptor.columnValue(H2RowDescriptor.java:338)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.getValue(GridH2AbstractKeyValueRow.java:236)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.compareRows(H2Tree.java:235)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:164)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:40)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.compare(BPlusTree.java:4359)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findInsertionPoint(BPlusTree.java:4279)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$1500(BPlusTree.java:81)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Search.run0(BPlusTree.java:261)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4697)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4682)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readPage(PageHandler.java:158)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.DataStructure.read(DataStructure.java:319)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1823)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1752)
~[ignite-core-2.1.4.jar:2.1.4]
... 22 more
{noformat}
was:
The following exceptions can be abserved during concurrent re-balancing and
cache destruction:
1.
{noformat}
[00:01:27,135][ERROR][sys-#4375%null%][GridDhtPreloader] Partition eviction
failed, this can cause grid hang.
org.apache.ignite.IgniteException: Runtime failure on search row: Row@6be51c3d[
**REMOVED SENSITIVE INFORMATION** ]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1787)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.remove(BPlusTree.java:1578)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex.remove(H2TreeIndex.java:226)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:523)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:416)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:574)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2172)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1462)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1425)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3224)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry.clearInternal(GridDhtCacheEntry.java:588)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.clearAll(GridDhtLocalPartition.java:951)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.tryEvict(GridDhtLocalPartition.java:809)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:593)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:580)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6629)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$2.body(GridClosureProcessor.java:967)
[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
[ignite-core-2.1.4.jar:2.1.4]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0_131]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0_131]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: java.lang.IllegalStateException: Item not found: 1
at
org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.findIndirectItemIndex(DataPageIO.java:346)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.getDataOffset(DataPageIO.java:446)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.readPayload(DataPageIO.java:488)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:149)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:101)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2RowFactory.getRow(H2RowFactory.java:62)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:60)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:33)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:123)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:40)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:4372)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:164)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:40)
~[ignite-indexing-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.compare(BPlusTree.java:4359)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findInsertionPoint(BPlusTree.java:4279)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$1500(BPlusTree.java:81)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Search.run0(BPlusTree.java:261)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4697)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4682)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readPage(PageHandler.java:158)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.DataStructure.read(DataStructure.java:319)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1823)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
~[ignite-core-2.1.4.jar:2.1.4]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1752)
~[ignite-core-2.1.4.jar:2.1.4]
... 22 more
{noformat}
> Cluster can fail during concurrent re-balancing and cache destruction
> ---------------------------------------------------------------------
>
> Key: IGNITE-6580
> URL: https://issues.apache.org/jira/browse/IGNITE-6580
> Project: Ignite
> Issue Type: Bug
> Components: cache
> Reporter: Mikhail Cherkasov
> Priority: Critical
>
> The following exceptions can be abserved during concurrent re-balancing and
> cache destruction:
> 1.
> {noformat}
> [00:01:27,135][ERROR][sys-#4375%null%][GridDhtPreloader] Partition eviction
> failed, this can cause grid hang.
> org.apache.ignite.IgniteException: Runtime failure on search row:
> Row@6be51c3d[ **REMOVED SENSITIVE INFORMATION** ]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1787)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.remove(BPlusTree.java:1578)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex.remove(H2TreeIndex.java:226)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:523)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:416)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:574)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2172)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1462)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1425)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3224)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry.clearInternal(GridDhtCacheEntry.java:588)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.clearAll(GridDhtLocalPartition.java:951)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.tryEvict(GridDhtLocalPartition.java:809)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:593)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:580)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6629)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.closure.GridClosureProcessor$2.body(GridClosureProcessor.java:967)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [?:1.8.0_131]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [?:1.8.0_131]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
> Caused by: java.lang.IllegalStateException: Item not found: 1
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.findIndirectItemIndex(DataPageIO.java:346)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.getDataOffset(DataPageIO.java:446)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.readPayload(DataPageIO.java:488)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:149)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:101)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2RowFactory.getRow(H2RowFactory.java:62)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:60)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:33)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:123)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:40)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:4372)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:164)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:40)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.compare(BPlusTree.java:4359)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findInsertionPoint(BPlusTree.java:4279)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$1500(BPlusTree.java:81)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Search.run0(BPlusTree.java:261)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4697)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4682)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readPage(PageHandler.java:158)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.DataStructure.read(DataStructure.java:319)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1823)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1752)
> ~[ignite-core-2.1.4.jar:2.1.4]
> ... 22 more
> {noformat}
> 2.
> {noformat}
> [00:01:27,454][ERROR][sys-#4375%null%][GridDhtPreloader] Partition eviction
> failed, this can cause grid hang.
> org.apache.ignite.IgniteException: Runtime failure on search row:
> Row@146ed070[ **REMOVED SENSITIVE INFORMATION**]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1787)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.remove(BPlusTree.java:1578)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex.remove(H2TreeIndex.java:226)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:523)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:416)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:574)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2172)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1462)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1425)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3224)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry.clearInternal(GridDhtCacheEntry.java:588)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.clearAll(GridDhtLocalPartition.java:951)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.tryEvict(GridDhtLocalPartition.java:809)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:593)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:580)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6629)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.closure.GridClosureProcessor$2.body(GridClosureProcessor.java:967)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
> [ignite-core-2.1.4.jar:2.1.4]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [?:1.8.0_131]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [?:1.8.0_131]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
> Caused by: org.apache.ignite.binary.BinaryObjectException: Failed to get
> field because type ID of passed object differs from type ID this BinaryField
> belongs to [expected=-1565418643, actual=-461073156]
> at
> org.apache.ignite.internal.binary.BinaryFieldImpl.fieldOrder(BinaryFieldImpl.java:274)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.binary.BinaryFieldImpl.value(BinaryFieldImpl.java:106)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.property.QueryBinaryProperty.fieldValue(QueryBinaryProperty.java:239)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.property.QueryBinaryProperty.value(QueryBinaryProperty.java:135)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.H2RowDescriptor.columnValue(H2RowDescriptor.java:338)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.getValue(GridH2AbstractKeyValueRow.java:236)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compareRows(H2Tree.java:235)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:164)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:40)
> ~[ignite-indexing-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.compare(BPlusTree.java:4359)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findInsertionPoint(BPlusTree.java:4279)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$1500(BPlusTree.java:81)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Search.run0(BPlusTree.java:261)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4697)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4682)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readPage(PageHandler.java:158)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.DataStructure.read(DataStructure.java:319)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1823)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
> ~[ignite-core-2.1.4.jar:2.1.4]
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1752)
> ~[ignite-core-2.1.4.jar:2.1.4]
> ... 22 more
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
