[
https://issues.apache.org/jira/browse/IGNITE-25503?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17956104#comment-17956104
]
Sergey Korotkov commented on IGNITE-25503:
------------------------------------------
Reproduced in the unit test both with 0 and non-zero "Tail not found"
exceptions in different stack traces. See below.
{noformat}
[19:34:47,012][SEVERE][pub-#121%remote-jvm-server%][IgniteCacheOffheapManagerImpl]
Failed to clear cache entry: KeyCacheObjectImpl [part=0, val=505,
hasValBytes=true]
class
org.apache.ignite.internal.processors.cache.persistence.freelist.CorruptedFreeListException:
Failed to remove data by link
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:839)
at
org.apache.ignite.internal.processors.cache.persistence.RowStore.removeRow(RowStore.java:89)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1766)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1736)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:420)
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3539)
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.clear(GridCacheMapEntry.java:1893)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.clearCache(IgniteCacheOffheapManagerImpl.java:478)
at
org.apache.ignite.internal.processors.cache.GridCacheClearAllRunnable.run(GridCacheClearAllRunnable.java:84)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.clearLocally(GridCacheAdapter.java:1046)
at
org.apache.ignite.internal.processors.cache.GridCacheProxyImpl.clearLocally(GridCacheProxyImpl.java:941)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter$GlobalClearAllJob.localExecute(GridCacheAdapter.java:5157)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter$TopologyVersionAwareJob.execute(GridCacheAdapter.java:6231)
at
org.apache.ignite.internal.processors.job.GridJobWorker$1.call(GridJobWorker.java:628)
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:5828)
at
org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:622)
at
org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:547)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at
org.apache.ignite.internal.processors.job.GridJobProcessor.runSync(GridJobProcessor.java:1475)
at
org.apache.ignite.internal.processors.job.GridJobProcessor.processJobExecuteRequest(GridJobProcessor.java:1388)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.sendRequest(GridTaskWorker.java:1441)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.processMappedJobs(GridTaskWorker.java:670)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.body(GridTaskWorker.java:534)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.startTask(GridTaskProcessor.java:760)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.execute(GridTaskProcessor.java:453)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.executeClearTask(GridCacheAdapter.java:1130)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.clear(GridCacheAdapter.java:1104)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.clear(GridCacheAdapter.java:1069)
at
org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.clear(IgniteCacheProxyImpl.java:1665)
at
org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.clear(GatewayProtectedCacheProxy.java:1138)
at
org.apache.ignite.internal.processors.database.FreeListMergeNoNextDifferentGcTest$TestMergeNoNextJob.run(FreeListMergeNoNextDifferentGcTest.java:230)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$C4.execute(GridClosureProcessor.java:1785)
at
org.apache.ignite.internal.processors.job.GridJobWorker$1.call(GridJobWorker.java:628)
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:5828)
at
org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:622)
at
org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:547)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at
org.apache.ignite.internal.processors.job.GridJobProcessor.processJobExecuteRequest(GridJobProcessor.java:1462)
at
org.apache.ignite.internal.processors.job.GridJobProcessor$JobExecutionListener.onMessage(GridJobProcessor.java:2261)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1880)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1501)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1405)
at
org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.IllegalStateException: Tail not found: 0
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.findTailIndex(PagesList.java:733)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.updateTail(PagesList.java:680)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList$CutTail.run(PagesList.java:166)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList$CutTail.run(PagesList.java:139)
at
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:313)
at
org.apache.ignite.internal.processors.cache.persistence.DataStructure.write(DataStructure.java:325)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.mergeNoNext(PagesList.java:1752)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.removeDataPage(PagesList.java:1707)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList$RemoveRowHandler.run(AbstractFreeList.java:405)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList$RemoveRowHandler.run(AbstractFreeList.java:359)
at
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:313)
at
org.apache.ignite.internal.processors.cache.persistence.DataStructure.write(DataStructure.java:325)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:815)
... 46 more
{noformat}
{noformat}
[19:34:47,022][SEVERE][pub-#121%remote-jvm-server%][IgniteCacheOffheapManagerImpl]
Failed to clear cache entry: KeyCacheObjectImpl [part=0, val=1009,
hasValBytes=true]
class
org.apache.ignite.internal.processors.cache.persistence.freelist.CorruptedFreeListException:
Failed to remove data by link
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:839)
at
org.apache.ignite.internal.processors.cache.persistence.RowStore.removeRow(RowStore.java:89)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1766)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1736)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:420)
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3539)
at
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.clear(GridCacheMapEntry.java:1893)
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.clearCache(IgniteCacheOffheapManagerImpl.java:478)
at
org.apache.ignite.internal.processors.cache.GridCacheClearAllRunnable.run(GridCacheClearAllRunnable.java:84)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.clearLocally(GridCacheAdapter.java:1046)
at
org.apache.ignite.internal.processors.cache.GridCacheProxyImpl.clearLocally(GridCacheProxyImpl.java:941)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter$GlobalClearAllJob.localExecute(GridCacheAdapter.java:5157)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter$TopologyVersionAwareJob.execute(GridCacheAdapter.java:6231)
at
org.apache.ignite.internal.processors.job.GridJobWorker$1.call(GridJobWorker.java:628)
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:5828)
at
org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:622)
at
org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:547)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at
org.apache.ignite.internal.processors.job.GridJobProcessor.runSync(GridJobProcessor.java:1475)
at
org.apache.ignite.internal.processors.job.GridJobProcessor.processJobExecuteRequest(GridJobProcessor.java:1388)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.sendRequest(GridTaskWorker.java:1441)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.processMappedJobs(GridTaskWorker.java:670)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.body(GridTaskWorker.java:534)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.startTask(GridTaskProcessor.java:760)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.execute(GridTaskProcessor.java:453)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.executeClearTask(GridCacheAdapter.java:1130)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.clear(GridCacheAdapter.java:1104)
at
org.apache.ignite.internal.processors.cache.GridCacheAdapter.clear(GridCacheAdapter.java:1069)
at
org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.clear(IgniteCacheProxyImpl.java:1665)
at
org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.clear(GatewayProtectedCacheProxy.java:1138)
at
org.apache.ignite.internal.processors.database.FreeListMergeNoNextDifferentGcTest$TestMergeNoNextJob.run(FreeListMergeNoNextDifferentGcTest.java:230)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor$C4.execute(GridClosureProcessor.java:1785)
at
org.apache.ignite.internal.processors.job.GridJobWorker$1.call(GridJobWorker.java:628)
at
org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:5828)
at
org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:622)
at
org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:547)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at
org.apache.ignite.internal.processors.job.GridJobProcessor.processJobExecuteRequest(GridJobProcessor.java:1462)
at
org.apache.ignite.internal.processors.job.GridJobProcessor$JobExecutionListener.onMessage(GridJobProcessor.java:2261)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1880)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1501)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1405)
at
org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.IllegalStateException: Tail not found: 844420635166727
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.findTailIndex(PagesList.java:733)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.updateTail(PagesList.java:680)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.mergeNoNext(PagesList.java:1758)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.removeDataPage(PagesList.java:1707)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList$RemoveRowHandler.run(AbstractFreeList.java:405)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList$RemoveRowHandler.run(AbstractFreeList.java:359)
at
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:313)
at
org.apache.ignite.internal.processors.cache.persistence.DataStructure.write(DataStructure.java:325)
at
org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:815)
... 46 more
{noformat}
> CorruptedFreeListException on cache clear calling mergeNoNext (jdk17,
> Shenandoah CG)
> ------------------------------------------------------------------------------------
>
> Key: IGNITE-25503
> URL: https://issues.apache.org/jira/browse/IGNITE-25503
> Project: Ignite
> Issue Type: Bug
> Reporter: Sergey Korotkov
> Assignee: Sergey Korotkov
> Priority: Major
> Labels: ise
> Time Spent: 10m
> Remaining Estimate: 0h
>
> * JDK17
> * Shenandoah GC
> * assertions OFF
> * in memory
> {{java.lang.IllegalStateException: Tail not found: 289074797055364410}}
> exception is thrown during cache clear.
> Possibly there are concurrent updates/deletes to other caches.
> This problem reproduced simultaneously with the one described in IGNITE-17734
> {noformat}
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl:
> Failed to clear cache entry: KeyCacheObjectImpl [part=753,
> val=FrseteFCj0mYYtuI66OG31uAFIdZdng0, hasValBytes=true]:
> org.apache.ignite.internal.proce
> ssors.cache.persistence.freelist.CorruptedFreeListException: Failed to remove
> data by link Caused by: java.lang.IllegalStateException: Tail not found:
> 289074797055364410
> class
> org.apache.ignite.internal.processors.cache.persistence.freelist.CorruptedFreeListException:
> Failed to remove data by link
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:839)
> at
> org.apache.ignite.internal.processors.cache.persistence.RowStore.removeRow(RowStore.java:89)
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1766)
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1736)
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:420)
> at
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3549)
> at
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.clear(GridCacheMapEntry.java:1903)
> at
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.clearCache(IgniteCacheOffheapManagerImpl.java:478)
> at
> org.apache.ignite.internal.processors.cache.GridCacheClearAllRunnable.run(GridCacheClearAllRunnable.java:84)
> at
> org.apache.ignite.internal.processors.cache.GridCacheAdapter.clearLocally(GridCacheAdapter.java:1039)
> at
> org.apache.ignite.internal.processors.cache.GridCacheProxyImpl.clearLocally(GridCacheProxyImpl.java:939)
> at
> org.apache.ignite.internal.processors.cache.GridCacheAdapter$GlobalClearAllJob.localExecute(GridCacheAdapter.java:5091)
> at
> org.apache.ignite.internal.processors.cache.GridCacheAdapter$TopologyVersionAwareJob.execute(GridCacheAdapter.java:6123)
> at
> org.apache.ignite.internal.processors.job.GridJobWorker$1.call(GridJobWorker.java:628)
> at
> org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:7474)
> at
> org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:622)
> at
> org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:547)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
> at
> org.apache.ignite.internal.processors.job.GridJobProcessor.processJobExecuteRequest(GridJobProcessor.java:1367)
> at
> org.apache.ignite.internal.processors.job.GridJobProcessor$JobExecutionListener.onMessage(GridJobProcessor.java:2223)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1893)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1514)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1418)
> at
> org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> at java.base/java.lang.Thread.run(Thread.java:840)
> Caused by: java.lang.IllegalStateException: Tail not found: 289074797055364410
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.findTailIndex(PagesList.java:731)
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.updateTail(PagesList.java:678)
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.mergeNoNext(PagesList.java:1756)
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList.removeDataPage(PagesList.java:1705)
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList$RemoveRowHandler.run(AbstractFreeList.java:405)
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList$RemoveRowHandler.run(AbstractFreeList.java:359)
> at
> org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:313)
> at
> org.apache.ignite.internal.processors.cache.persistence.DataStructure.write(DataStructure.java:325)
> at
> org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:815)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
