[
https://issues.apache.org/jira/browse/IGNITE-12456?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16997900#comment-16997900
]
Ravi Kumar Powli commented on IGNITE-12456:
-------------------------------------------
Please find the below thread dump logs from client Nodes.
"grid-timeout-worker-#23%DataStoreIgniteCache%" #50 daemon prio=5 os_prio=0
tid=0x00007f3284fb5000 nid=0x105d in Object.wait()
[0x00007f3280ae7000]"grid-timeout-worker-#23%DataStoreIgniteCache%" #50 daemon
prio=5 os_prio=0 tid=0x00007f3284fb5000 nid=0x105d in Object.wait()
[0x00007f3280ae7000] java.lang.Thread.State: TIMED_WAITING (on object
monitor) at java.lang.Object.wait(Native Method) at
org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:269)
- locked <0x000000047e298de0> (a java.lang.Object) at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
java.lang.Thread.run(Thread.java:748)
"ignite-clock" #49 daemon prio=10 os_prio=0 tid=0x00007f3286bc6800 nid=0x105c
sleeping[0x00007f3280be8000] java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method) at
org.apache.ignite.internal.util.IgniteUtils$22.run(IgniteUtils.java:3379) at
java.lang.Thread.run(Thread.java:748)
"jvm-pause-detector-worker" #42 daemon prio=5 os_prio=0 tid=0x00007f3286b73800
nid=0x1055 waiting on condition [0x00007f32814ef000] java.lang.Thread.State:
TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at
org.apache.ignite.internal.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:100)
"data-streamer-stripe-7-#16%DataStoreIgniteCache%" #41 daemon prio=5 os_prio=0
tid=0x00007f3286191000 nid=0x1054 waiting on condition [0x00007f32815f0000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"data-streamer-stripe-6-#15%DataStoreIgniteCache%" #40 daemon prio=5 os_prio=0
tid=0x00007f328618f000 nid=0x1053 waiting on condition [0x00007f32816f1000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"data-streamer-stripe-5-#14%DataStoreIgniteCache%" #39 daemon prio=5 os_prio=0
tid=0x00007f328618d000 nid=0x1052 waiting on condition [0x00007f32817f2000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"data-streamer-stripe-4-#13%DataStoreIgniteCache%" #38 daemon prio=5 os_prio=0
tid=0x00007f328618b000 nid=0x1051 waiting on condition [0x00007f32818f3000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"data-streamer-stripe-3-#12%DataStoreIgniteCache%" #37 daemon prio=5 os_prio=0
tid=0x00007f3286189000 nid=0x1050 waiting on condition [0x00007f32819f4000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"data-streamer-stripe-2-#11%DataStoreIgniteCache%" #36 daemon prio=5 os_prio=0
tid=0x00007f3286187800 nid=0x104f waiting on condition [0x00007f3281af5000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"data-streamer-stripe-1-#10%DataStoreIgniteCache%" #35 daemon prio=5 os_prio=0
tid=0x00007f3286185800 nid=0x104e waiting on condition [0x00007f3281bf6000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"data-streamer-stripe-0-#9%DataStoreIgniteCache%" #34 daemon prio=5 os_prio=0
tid=0x00007f3286184800 nid=0x104d waiting on condition [0x00007f3281cf7000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-7-#8%DataStoreIgniteCache%" #33 daemon prio=5 os_prio=0
tid=0x00007f3286182000 nid=0x104c waiting on condition [0x00007f3281df8000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-6-#7%DataStoreIgniteCache%" #32 daemon prio=5 os_prio=0
tid=0x00007f3286180000 nid=0x104b waiting on condition [0x00007f3281ef9000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-5-#6%DataStoreIgniteCache%" #31 daemon prio=5 os_prio=0
tid=0x00007f328617e800 nid=0x104a waiting on condition [0x00007f3281ffa000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-4-#5%DataStoreIgniteCache%" #30 daemon prio=5 os_prio=0
tid=0x00007f328617c800 nid=0x1049 waiting on condition [0x00007f32820fb000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-3-#4%DataStoreIgniteCache%" #29 daemon prio=5 os_prio=0
tid=0x00007f328617b000 nid=0x1048 waiting on condition [0x00007f32821fc000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-2-#3%DataStoreIgniteCache%" #28 daemon prio=5 os_prio=0
tid=0x00007f3286179800 nid=0x1047 waiting on condition [0x00007f32822fd000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-1-#2%DataStoreIgniteCache%" #27 daemon prio=5 os_prio=0
tid=0x00007f3286762000 nid=0x1046 waiting on condition [0x00007f32823fe000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
"sys-stripe-0-#1%DataStoreIgniteCache%" #26 daemon prio=5 os_prio=0
tid=0x00007f3286761000 nid=0x1045 waiting on condition [0x00007f32941fc000]
java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native
Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at
org.apache.ignite.internal.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:493)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
> Cluster Data Store grid gets Corrupted for Load test
> ----------------------------------------------------
>
> Key: IGNITE-12456
> URL: https://issues.apache.org/jira/browse/IGNITE-12456
> Project: Ignite
> Issue Type: Bug
> Components: general
> Affects Versions: 2.7
> Reporter: Ravi Kumar Powli
> Priority: Blocker
> Fix For: 2.8
>
> Attachments: default-config.xml
>
>
> We have Apache Ignite 3 node cluster setup with Amazon S3 Based Discovery.we
> are into AWS cloud environment with Microservice model with 8
> microservices.we are using Ignite for session data store.While performing
> load test we are facing data grid issues if the number of clients reached 40
> , Once data grid got corrupted we will lost the session store data and
> Application will not respond due to session data also corrupted and all the
> instances will auto scaled down to initial size 8.We need to restart Apache
> Ignite to make the Application up.Please find the attached Apache Ignite
> Configuration for your reference.
> Apache Ignite Version : 2.7.0
> 07:24:46,678][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-5,
> blockedFor=21s]07:24:46,678][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-5,
> blockedFor=21s][07:24:46,680][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-5, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271465499]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-5, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271465499] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:24:52,692][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=ttl-cleanup-worker,
> blockedFor=27s][07:24:52,692][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=ttl-cleanup-worker, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271465044]]]class
> org.apache.ignite.IgniteException: GridWorker [name=ttl-cleanup-worker,
> igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271465044] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:24:55,694][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-1,
> blockedFor=24s][07:24:55,694][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-1, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271471538]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-1, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271471538] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:24:58,700][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-4,
> blockedFor=29s][07:24:58,700][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-4, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271469397]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-4, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271469397] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:24:59,833][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-7,
> blockedFor=29s][07:24:59,833][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-7, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271469973]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-7, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271469973] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:25:04,716][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-0,
> blockedFor=39s][07:25:04,716][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-0, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271464782]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-0, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271464782] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:25:05,870][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-3,
> blockedFor=34s][07:25:05,870][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-3, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271471528]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-3, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271471528] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:25:07,011][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-6,
> blockedFor=41s][07:25:07,011][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-6, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271465428]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-6, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271465428] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:25:08,167][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-5,
> blockedFor=42s][07:25:08,167][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-5, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271465499]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-5, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271465499] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:25:09,310][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=sys-stripe-2,
> blockedFor=40s][07:25:09,311][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=sys-stripe-2, igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271468397]]]class org.apache.ignite.IgniteException:
> GridWorker [name=sys-stripe-2, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271468397] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at
> org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)[07:25:14,183][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][G]
> Blocked system-critical thread has been detected. This can lead to
> cluster-wide undefined behaviour [threadName=ttl-cleanup-worker,
> blockedFor=49s][07:25:14,183][SEVERE][tcp-disco-msg-worker-#2%DataStoreIgniteCache%][]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED]]], failureCtx=FailureContext
> [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker
> [name=ttl-cleanup-worker, igniteInstanceName=DataStoreIgniteCache,
> finished=false, heartbeatTs=1575271465044]]]class
> org.apache.ignite.IgniteException: GridWorker [name=ttl-cleanup-worker,
> igniteInstanceName=DataStoreIgniteCache, finished=false,
> heartbeatTs=1575271465044] at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1826)
> at
> org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233)
> at
> org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2663)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7181)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2700)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7119)
> at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
