[ https://issues.apache.org/jira/browse/IGNITE-15961?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Maxim Muzafarov updated IGNITE-15961: ------------------------------------- Description: The test {{>>> CacheGroupKeyChangeTest#testNodeWithOlderKeyBecameCoordinator <<<}} fails with o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted. See the full log for details. https://ci.ignite.apache.org/downloadBuildLog.html?buildId=6278078 {code} [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 21:15:45,134][ERROR][rebalance-striped-#27523%grid-1%][IgniteTestResources] Critical system error detected. Will be handled accordingly to configured handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=1613773252, pageIds=[], msg=Runtime failure on bounds: [lower=null, upper=null]]]] [21:15:45]W: [org.apache.ignite:ignite-core] class org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=1613773252, pageIds=[], msg=Runtime failure on bounds: [lower=null, upper=null]] [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6162) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1124) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:168) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:63) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1088) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2897) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.cursor(GridCacheOffheapManager.java:2890) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$6.onHasNext(IgniteCacheOffheapManagerImpl.java:1161) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:81) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:144) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:215) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.peek(IgniteRebalanceIteratorImpl.java:155) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:330) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.lambda$handleDemandMessage$1(GridDhtPreloader.java:361) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [21:15:45]W: [org.apache.ignite:ignite-core] at java.lang.Thread.run(Thread.java:748) [21:15:45]W: [org.apache.ignite:ignite-core] Caused by: org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTreeRuntimeException: java.lang.IllegalMonitorStateException: Attempted to release a read lock while not holding it [lock=0000557bc43771f8, state=000000000001ffff] [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findLowerUnbounded(BPlusTree.java:1067) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1106) [21:15:45]W: [org.apache.ignite:ignite-core] ... 16 more [21:15:45]W: [org.apache.ignite:ignite-core] Caused by: java.lang.IllegalMonitorStateException: Attempted to release a read lock while not holding it [lock=0000557bc43771f8, state=000000000001ffff] [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.util.OffheapReadWriteLock.readUnlock(OffheapReadWriteLock.java:172) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.readUnlockPage(PageMemoryImpl.java:1592) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.readUnlock(PageMemoryImpl.java:490) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readUnlock(PageHandler.java:222) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.readUnlock(DataStructure.java:265) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findLowerUnbounded(BPlusTree.java:1059) [21:15:45]W: [org.apache.ignite:ignite-core] ... 17 more [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 21:15:45,144][ERROR][rebalance-striped-#27523%grid-1%][FailureProcessor] A critical problem with persistence data structures was detected. Please make backup of persistence storage and WAL files for further analysis. Persistence storage path: null WAL path: db/wal WAL archive path: db/wal/archive [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 21:15:45,145][ERROR][rebalance-striped-#27523%grid-1%][GridDhtPartitionSupplier] Failed to continue supplying [grp=encrypted, demander=00a6fc03-d771-41c5-a569-486fa9959000, topVer=AffinityTopologyVersion [topVer=2, minorTopVer=1]] [21:15:45]W: [org.apache.ignite:ignite-core] java.lang.AssertionError [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.toList(LockLog.java:156) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.snapshot(LockLog.java:134) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.snapshot(LockLog.java:29) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTracker.dump(PageLockTracker.java:355) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.SharedPageLockTracker.dump(SharedPageLockTracker.java:209) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocks(PageLockTrackerManager.java:175) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocksToString(PageLockTrackerManager.java:184) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocksToLog(PageLockTrackerManager.java:191) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.onFailure(DiagnosticProcessor.java:96) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.failure.FailureProcessor.process(FailureProcessor.java:210) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.failure.FailureProcessor.process(FailureProcessor.java:156) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.processFailure(BPlusTree.java:6177) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6164) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1124) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:168) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:63) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1088) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2897) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.cursor(GridCacheOffheapManager.java:2890) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$6.onHasNext(IgniteCacheOffheapManagerImpl.java:1161) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:81) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:144) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:215) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.peek(IgniteRebalanceIteratorImpl.java:155) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:330) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.lambda$handleDemandMessage$1(GridDhtPreloader.java:361) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [21:15:45]W: [org.apache.ignite:ignite-core] at java.lang.Thread.run(Thread.java:748) {code} was: The test {{>>> CacheGroupKeyChangeTest#testNodeWithOlderKeyBecameCoordinator <<<}} fails with o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted. See the full log for details. https://ci.ignite.apache.org/buildConfiguration/IgniteTests24Java8_Basic3/6278078?buildTab=log&focusLine=3 {code} [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 21:15:45,134][ERROR][rebalance-striped-#27523%grid-1%][IgniteTestResources] Critical system error detected. Will be handled accordingly to configured handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=1613773252, pageIds=[], msg=Runtime failure on bounds: [lower=null, upper=null]]]] [21:15:45]W: [org.apache.ignite:ignite-core] class org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=1613773252, pageIds=[], msg=Runtime failure on bounds: [lower=null, upper=null]] [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6162) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1124) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:168) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:63) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1088) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2897) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.cursor(GridCacheOffheapManager.java:2890) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$6.onHasNext(IgniteCacheOffheapManagerImpl.java:1161) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:81) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:144) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:215) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.peek(IgniteRebalanceIteratorImpl.java:155) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:330) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.lambda$handleDemandMessage$1(GridDhtPreloader.java:361) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [21:15:45]W: [org.apache.ignite:ignite-core] at java.lang.Thread.run(Thread.java:748) [21:15:45]W: [org.apache.ignite:ignite-core] Caused by: org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTreeRuntimeException: java.lang.IllegalMonitorStateException: Attempted to release a read lock while not holding it [lock=0000557bc43771f8, state=000000000001ffff] [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findLowerUnbounded(BPlusTree.java:1067) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1106) [21:15:45]W: [org.apache.ignite:ignite-core] ... 16 more [21:15:45]W: [org.apache.ignite:ignite-core] Caused by: java.lang.IllegalMonitorStateException: Attempted to release a read lock while not holding it [lock=0000557bc43771f8, state=000000000001ffff] [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.util.OffheapReadWriteLock.readUnlock(OffheapReadWriteLock.java:172) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.readUnlockPage(PageMemoryImpl.java:1592) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.readUnlock(PageMemoryImpl.java:490) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readUnlock(PageHandler.java:222) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.readUnlock(DataStructure.java:265) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findLowerUnbounded(BPlusTree.java:1059) [21:15:45]W: [org.apache.ignite:ignite-core] ... 17 more [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 21:15:45,144][ERROR][rebalance-striped-#27523%grid-1%][FailureProcessor] A critical problem with persistence data structures was detected. Please make backup of persistence storage and WAL files for further analysis. Persistence storage path: null WAL path: db/wal WAL archive path: db/wal/archive [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 21:15:45,145][ERROR][rebalance-striped-#27523%grid-1%][GridDhtPartitionSupplier] Failed to continue supplying [grp=encrypted, demander=00a6fc03-d771-41c5-a569-486fa9959000, topVer=AffinityTopologyVersion [topVer=2, minorTopVer=1]] [21:15:45]W: [org.apache.ignite:ignite-core] java.lang.AssertionError [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.toList(LockLog.java:156) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.snapshot(LockLog.java:134) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.snapshot(LockLog.java:29) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTracker.dump(PageLockTracker.java:355) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.SharedPageLockTracker.dump(SharedPageLockTracker.java:209) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocks(PageLockTrackerManager.java:175) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocksToString(PageLockTrackerManager.java:184) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocksToLog(PageLockTrackerManager.java:191) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.onFailure(DiagnosticProcessor.java:96) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.failure.FailureProcessor.process(FailureProcessor.java:210) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.failure.FailureProcessor.process(FailureProcessor.java:156) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.processFailure(BPlusTree.java:6177) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6164) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1124) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:168) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:63) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1088) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2897) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.cursor(GridCacheOffheapManager.java:2890) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$6.onHasNext(IgniteCacheOffheapManagerImpl.java:1161) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:81) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:144) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:215) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.peek(IgniteRebalanceIteratorImpl.java:155) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:330) [21:15:45]W: [org.apache.ignite:ignite-core] at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.lambda$handleDemandMessage$1(GridDhtPreloader.java:361) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [21:15:45]W: [org.apache.ignite:ignite-core] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [21:15:45]W: [org.apache.ignite:ignite-core] at java.lang.Thread.run(Thread.java:748) {code} > CacheGroupKeyChangeTest#testNodeWithOlderKeyBecameCoordinator fails with > CorruptedTreeException (B+Tree is corrupted) > --------------------------------------------------------------------------------------------------------------------- > > Key: IGNITE-15961 > URL: https://issues.apache.org/jira/browse/IGNITE-15961 > Project: Ignite > Issue Type: Bug > Reporter: Maxim Muzafarov > Priority: Blocker > Attachments: Ignite_Tests_2.x_JDK_8_11_Basic_3_95.log > > > The test {{>>> CacheGroupKeyChangeTest#testNodeWithOlderKeyBecameCoordinator > <<<}} fails with > o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is > corrupted. See the full log for details. > https://ci.ignite.apache.org/downloadBuildLog.html?buildId=6278078 > {code} > [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 > 21:15:45,134][ERROR][rebalance-striped-#27523%grid-1%][IgniteTestResources] > Critical system error detected. Will be handled accordingly to configured > handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler > [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, > SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext > [type=CRITICAL_ERROR, err=class > o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is > corrupted [groupId=1613773252, pageIds=[], msg=Runtime failure on bounds: > [lower=null, upper=null]]]] > [21:15:45]W: [org.apache.ignite:ignite-core] class > org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException: > B+Tree is corrupted [groupId=1613773252, pageIds=[], msg=Runtime failure on > bounds: [lower=null, upper=null]] > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6162) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1124) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:168) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:63) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1088) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2897) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.cursor(GridCacheOffheapManager.java:2890) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$6.onHasNext(IgniteCacheOffheapManagerImpl.java:1161) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:81) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:144) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:215) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.peek(IgniteRebalanceIteratorImpl.java:155) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:330) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.lambda$handleDemandMessage$1(GridDhtPreloader.java:361) > [21:15:45]W: [org.apache.ignite:ignite-core] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > [21:15:45]W: [org.apache.ignite:ignite-core] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > [21:15:45]W: [org.apache.ignite:ignite-core] at > java.lang.Thread.run(Thread.java:748) > [21:15:45]W: [org.apache.ignite:ignite-core] Caused by: > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTreeRuntimeException: > java.lang.IllegalMonitorStateException: Attempted to release a read lock > while not holding it [lock=0000557bc43771f8, state=000000000001ffff] > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findLowerUnbounded(BPlusTree.java:1067) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1106) > [21:15:45]W: [org.apache.ignite:ignite-core] ... 16 more > [21:15:45]W: [org.apache.ignite:ignite-core] Caused by: > java.lang.IllegalMonitorStateException: Attempted to release a read lock > while not holding it [lock=0000557bc43771f8, state=000000000001ffff] > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.util.OffheapReadWriteLock.readUnlock(OffheapReadWriteLock.java:172) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.readUnlockPage(PageMemoryImpl.java:1592) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.readUnlock(PageMemoryImpl.java:490) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.readUnlock(PageHandler.java:222) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.DataStructure.readUnlock(DataStructure.java:265) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findLowerUnbounded(BPlusTree.java:1059) > [21:15:45]W: [org.apache.ignite:ignite-core] ... 17 more > [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 > 21:15:45,144][ERROR][rebalance-striped-#27523%grid-1%][FailureProcessor] A > critical problem with persistence data structures was detected. Please make > backup of persistence storage and WAL files for further analysis. Persistence > storage path: null WAL path: db/wal WAL archive path: db/wal/archive > [21:15:45]W: [org.apache.ignite:ignite-core] [2021-11-19 > 21:15:45,145][ERROR][rebalance-striped-#27523%grid-1%][GridDhtPartitionSupplier] > Failed to continue supplying [grp=encrypted, > demander=00a6fc03-d771-41c5-a569-486fa9959000, topVer=AffinityTopologyVersion > [topVer=2, minorTopVer=1]] > [21:15:45]W: [org.apache.ignite:ignite-core] > java.lang.AssertionError > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.toList(LockLog.java:156) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.snapshot(LockLog.java:134) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.LockLog.snapshot(LockLog.java:29) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTracker.dump(PageLockTracker.java:355) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.SharedPageLockTracker.dump(SharedPageLockTracker.java:209) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocks(PageLockTrackerManager.java:175) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocksToString(PageLockTrackerManager.java:184) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager.dumpLocksToLog(PageLockTrackerManager.java:191) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.onFailure(DiagnosticProcessor.java:96) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.failure.FailureProcessor.process(FailureProcessor.java:210) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.failure.FailureProcessor.process(FailureProcessor.java:156) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.processFailure(BPlusTree.java:6177) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6164) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1124) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:168) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:63) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1088) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2897) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.cursor(GridCacheOffheapManager.java:2890) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$6.onHasNext(IgniteCacheOffheapManagerImpl.java:1161) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:81) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:144) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:215) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.peek(IgniteRebalanceIteratorImpl.java:155) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:330) > [21:15:45]W: [org.apache.ignite:ignite-core] at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.lambda$handleDemandMessage$1(GridDhtPreloader.java:361) > [21:15:45]W: [org.apache.ignite:ignite-core] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > [21:15:45]W: [org.apache.ignite:ignite-core] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > [21:15:45]W: [org.apache.ignite:ignite-core] at > java.lang.Thread.run(Thread.java:748) > {code} -- This message was sent by Atlassian Jira (v8.20.1#820001)