[ 
https://issues.apache.org/jira/browse/IGNITE-6580?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Alexey Goncharuk updated IGNITE-6580:
-------------------------------------
    Fix Version/s:     (was: 2.5)
                   2.6

> Cluster can fail during concurrent re-balancing and cache destruction
> ---------------------------------------------------------------------
>
>                 Key: IGNITE-6580
>                 URL: https://issues.apache.org/jira/browse/IGNITE-6580
>             Project: Ignite
>          Issue Type: Bug
>          Components: cache
>    Affects Versions: 2.2
>            Reporter: Mikhail Cherkasov
>            Assignee: Alexey Goncharuk
>            Priority: Major
>             Fix For: 2.6
>
>
> The following exceptions can be abserved during concurrent re-balancing and 
> cache destruction:
> 1.
> {noformat}
> [00:01:27,135][ERROR][sys-#4375%null%][GridDhtPreloader] Partition eviction 
> failed, this can cause grid hang.
> org.apache.ignite.IgniteException: Runtime failure on search row: 
> Row@6be51c3d[ **REMOVED SENSITIVE INFORMATION** ]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1787)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.remove(BPlusTree.java:1578)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex.remove(H2TreeIndex.java:226)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:523)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:416)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:574)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2172)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1462)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1425)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3224)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry.clearInternal(GridDhtCacheEntry.java:588)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.clearAll(GridDhtLocalPartition.java:951)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.tryEvict(GridDhtLocalPartition.java:809)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:593)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:580)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6629)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.closure.GridClosureProcessor$2.body(GridClosureProcessor.java:967)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) 
> [ignite-core-2.1.4.jar:2.1.4]
>       at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  [?:1.8.0_131]
>       at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  [?:1.8.0_131]
>       at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
> Caused by: java.lang.IllegalStateException: Item not found: 1
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.findIndirectItemIndex(DataPageIO.java:346)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.getDataOffset(DataPageIO.java:446)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.readPayload(DataPageIO.java:488)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:149)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:101)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2RowFactory.getRow(H2RowFactory.java:62)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:60)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO.getLookupRow(H2InnerIO.java:33)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:123)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.getRow(H2Tree.java:40)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:4372)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:164)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:40)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.compare(BPlusTree.java:4359)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findInsertionPoint(BPlusTree.java:4279)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$1500(BPlusTree.java:81)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Search.run0(BPlusTree.java:261)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4697)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4682)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readPage(PageHandler.java:158)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.DataStructure.read(DataStructure.java:319)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1823)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1752)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       ... 22 more
> {noformat}
> 2.
> {noformat}
> [00:01:27,454][ERROR][sys-#4375%null%][GridDhtPreloader] Partition eviction 
> failed, this can cause grid hang.
> org.apache.ignite.IgniteException: Runtime failure on search row: 
> Row@146ed070[ **REMOVED SENSITIVE INFORMATION**]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1787)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.remove(BPlusTree.java:1578)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex.remove(H2TreeIndex.java:226)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:523)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:416)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:574)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2172)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1462)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1425)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3224)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry.clearInternal(GridDhtCacheEntry.java:588)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.clearAll(GridDhtLocalPartition.java:951)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition.tryEvict(GridDhtLocalPartition.java:809)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:593)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$3.call(GridDhtPreloader.java:580)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6629)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.closure.GridClosureProcessor$2.body(GridClosureProcessor.java:967)
>  [ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) 
> [ignite-core-2.1.4.jar:2.1.4]
>       at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>  [?:1.8.0_131]
>       at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>  [?:1.8.0_131]
>       at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
> Caused by: org.apache.ignite.binary.BinaryObjectException: Failed to get 
> field because type ID of passed object differs from type ID this BinaryField 
> belongs to [expected=-1565418643, actual=-461073156]
>       at 
> org.apache.ignite.internal.binary.BinaryFieldImpl.fieldOrder(BinaryFieldImpl.java:274)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.binary.BinaryFieldImpl.value(BinaryFieldImpl.java:106)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.property.QueryBinaryProperty.fieldValue(QueryBinaryProperty.java:239)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.property.QueryBinaryProperty.value(QueryBinaryProperty.java:135)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.H2RowDescriptor.columnValue(H2RowDescriptor.java:338)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.getValue(GridH2AbstractKeyValueRow.java:236)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compareRows(H2Tree.java:235)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:164)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.query.h2.database.H2Tree.compare(H2Tree.java:40)
>  ~[ignite-indexing-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.compare(BPlusTree.java:4359)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findInsertionPoint(BPlusTree.java:4279)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$1500(BPlusTree.java:81)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Search.run0(BPlusTree.java:261)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4697)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:4682)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readPage(PageHandler.java:158)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.DataStructure.read(DataStructure.java:319)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1823)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.removeDown(BPlusTree.java:1842)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       at 
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doRemove(BPlusTree.java:1752)
>  ~[ignite-core-2.1.4.jar:2.1.4]
>       ... 22 more
> {noformat}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to