[ https://issues.apache.org/jira/browse/IGNITE-5793?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ivan Rakov updated IGNITE-5793: ------------------------------- Description: Right after expiration time, all threads from sys-stripe pool are busy with removing expired entries: {noformat} Thread [name="sys-stripe-3-#35%database.IgniteDbSnapshotWithEvictionsSelfTest1%", id=60, state=RUNNABLE, blockCnt=0, waitCnt=101794] at o.a.i.i.binary.BinaryObjectImpl.typeId(BinaryObjectImpl.java:278) at o.a.i.i.processors.cache.binary.CacheObjectBinaryProcessorImpl.typeId(CacheObjectBinaryProcessorImpl.java:672) at o.a.i.i.processors.query.GridQueryProcessor.typeByValue(GridQueryProcessor.java:1688) at o.a.i.i.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2177) at o.a.i.i.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1456) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1419) at o.a.i.i.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.remove(GridCacheOffheapManager.java:1241) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383) at o.a.i.i.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3221) at o.a.i.i.processors.cache.GridCacheMapEntry.onExpired(GridCacheMapEntry.java:3028) at o.a.i.i.processors.cache.GridCacheMapEntry.onTtlExpired(GridCacheMapEntry.java:2961) at o.a.i.i.processors.cache.GridCacheTtlManager$1.applyx(GridCacheTtlManager.java:61) at o.a.i.i.processors.cache.GridCacheTtlManager$1.applyx(GridCacheTtlManager.java:52) at o.a.i.i.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:38) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.expire(IgniteCacheOffheapManagerImpl.java:1007) at o.a.i.i.processors.cache.GridCacheTtlManager.expire(GridCacheTtlManager.java:198) at o.a.i.i.processors.cache.GridCacheTtlManager.expire(GridCacheTtlManager.java:160) at o.a.i.i.processors.cache.GridCacheUtils.unwindEvicts(GridCacheUtils.java:854) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1073) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:561) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184) at o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) at o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1097) at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:483) at java.lang.Thread.run(Thread.java:745) {noformat} System totally stops responding to user get/put/etc operations. The freeze can last for several checkpoints. Most likely, system fills cache with new entries faster than expiring old. As a result, old entries pile up. was: Right after expiration time, all threads from sys-stripe pool are busy with removing expired entries: {noformat} Thread [name="sys-stripe-3-#35%database.IgniteDbSnapshotWithEvictionsSelfTest1%", id=60, state=RUNNABLE, blockCnt=0, waitCnt=101794] at o.a.i.i.binary.BinaryObjectImpl.typeId(BinaryObjectImpl.java:278) at o.a.i.i.processors.cache.binary.CacheObjectBinaryProcessorImpl.typeId(CacheObjectBinaryProcessorImpl.java:672) at o.a.i.i.processors.query.GridQueryProcessor.typeByValue(GridQueryProcessor.java:1688) at o.a.i.i.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2177) at o.a.i.i.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1456) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1419) at o.a.i.i.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.remove(GridCacheOffheapManager.java:1241) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383) at o.a.i.i.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3221) at o.a.i.i.processors.cache.GridCacheMapEntry.onExpired(GridCacheMapEntry.java:3028) at o.a.i.i.processors.cache.GridCacheMapEntry.onTtlExpired(GridCacheMapEntry.java:2961) at o.a.i.i.processors.cache.GridCacheTtlManager$1.applyx(GridCacheTtlManager.java:61) at o.a.i.i.processors.cache.GridCacheTtlManager$1.applyx(GridCacheTtlManager.java:52) at o.a.i.i.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:38) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.expire(IgniteCacheOffheapManagerImpl.java:1007) at o.a.i.i.processors.cache.GridCacheTtlManager.expire(GridCacheTtlManager.java:198) at o.a.i.i.processors.cache.GridCacheTtlManager.expire(GridCacheTtlManager.java:160) at o.a.i.i.processors.cache.GridCacheUtils.unwindEvicts(GridCacheUtils.java:854) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1073) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:561) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184) at o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) at o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1097) at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:483) at java.lang.Thread.run(Thread.java:745) {noformat} System totally stops responding to user get/put/etc operation. The freeze can last for several checkpoints. > Cache with constant time TTL for entries and enabled persistence hangs for a > long time when TTL expirations start > ----------------------------------------------------------------------------------------------------------------- > > Key: IGNITE-5793 > URL: https://issues.apache.org/jira/browse/IGNITE-5793 > Project: Ignite > Issue Type: Bug > Components: cache > Affects Versions: 2.0 > Reporter: Ivan Rakov > Fix For: 2.2 > > > Right after expiration time, all threads from sys-stripe pool are busy with > removing expired entries: > {noformat} > Thread > [name="sys-stripe-3-#35%database.IgniteDbSnapshotWithEvictionsSelfTest1%", > id=60, state=RUNNABLE, blockCnt=0, waitCnt=101794] > at o.a.i.i.binary.BinaryObjectImpl.typeId(BinaryObjectImpl.java:278) > at > o.a.i.i.processors.cache.binary.CacheObjectBinaryProcessorImpl.typeId(CacheObjectBinaryProcessorImpl.java:672) > at > o.a.i.i.processors.query.GridQueryProcessor.typeByValue(GridQueryProcessor.java:1688) > at > o.a.i.i.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:2177) > at > o.a.i.i.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:451) > at > o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishRemove(IgniteCacheOffheapManagerImpl.java:1456) > at > o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.remove(IgniteCacheOffheapManagerImpl.java:1419) > at > o.a.i.i.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.remove(GridCacheOffheapManager.java:1241) > at > o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.remove(IgniteCacheOffheapManagerImpl.java:383) > at > o.a.i.i.processors.cache.GridCacheMapEntry.removeValue(GridCacheMapEntry.java:3221) > at > o.a.i.i.processors.cache.GridCacheMapEntry.onExpired(GridCacheMapEntry.java:3028) > at > o.a.i.i.processors.cache.GridCacheMapEntry.onTtlExpired(GridCacheMapEntry.java:2961) > at > o.a.i.i.processors.cache.GridCacheTtlManager$1.applyx(GridCacheTtlManager.java:61) > at > o.a.i.i.processors.cache.GridCacheTtlManager$1.applyx(GridCacheTtlManager.java:52) > at > o.a.i.i.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:38) > at > o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.expire(IgniteCacheOffheapManagerImpl.java:1007) > at > o.a.i.i.processors.cache.GridCacheTtlManager.expire(GridCacheTtlManager.java:198) > at > o.a.i.i.processors.cache.GridCacheTtlManager.expire(GridCacheTtlManager.java:160) > at > o.a.i.i.processors.cache.GridCacheUtils.unwindEvicts(GridCacheUtils.java:854) > at > o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1073) > at > o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:561) > at > o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) > at > o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) > at > o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) > at > o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) > at > o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556) > at > o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184) > at > o.a.i.i.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) > at > o.a.i.i.managers.communication.GridIoManager$9.run(GridIoManager.java:1097) > at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:483) > at java.lang.Thread.run(Thread.java:745) > {noformat} > System totally stops responding to user get/put/etc operations. The freeze > can last for several checkpoints. Most likely, system fills cache with new > entries faster than expiring old. As a result, old entries pile up. -- This message was sent by Atlassian JIRA (v6.4.14#64029)