[ https://issues.apache.org/jira/browse/CASSANDRA-13235?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15873460#comment-15873460 ]
zhaoyan edited comment on CASSANDRA-13235 at 2/20/17 2:49 AM: -------------------------------------------------------------- 383 is BLOCKED. only one is running: {code} "SharedPool-Worker-27" #415 daemon prio=5 os_prio=0 tid=0x000000000242d800 nid=0x744e runnable [0x00007fdd8372c000] java.lang.Thread.State: RUNNABLE at java.util.Arrays.copyOf(Arrays.java:3284) at org.apache.cassandra.db.RangeTombstoneList.copy(RangeTombstoneList.java:116) at org.apache.cassandra.db.DeletionInfo.copy(DeletionInfo.java:104) at org.apache.cassandra.db.AtomicBTreeColumns.addAllWithSizeDelta(AtomicBTreeColumns.java:217) at org.apache.cassandra.db.Memtable.put(Memtable.java:210) at org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1244) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:396) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:359) at org.apache.cassandra.db.Mutation.apply(Mutation.java:214) at org.apache.cassandra.db.MutationVerbHandler.doVerb(MutationVerbHandler.java:54) at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) at java.lang.Thread.run(Thread.java:745) {code} OR {code} "SharedPool-Worker-80" #585 daemon prio=5 os_prio=0 tid=0x00007f5f68993000 nid=0x4872 runnable [0x00007f5c6ce8d000] java.lang.Thread.State: RUNNABLE at org.apache.cassandra.db.RangeTombstoneList.grow(RangeTombstoneList.java:739) at org.apache.cassandra.db.RangeTombstoneList.grow(RangeTombstoneList.java:729) at org.apache.cassandra.db.RangeTombstoneList.growToFree(RangeTombstoneList.java:714) at org.apache.cassandra.db.RangeTombstoneList.addInternal(RangeTombstoneList.java:686) at org.apache.cassandra.db.RangeTombstoneList.insertFrom(RangeTombstoneList.java:602) at org.apache.cassandra.db.RangeTombstoneList.add(RangeTombstoneList.java:171) at org.apache.cassandra.db.RangeTombstoneList.addAll(RangeTombstoneList.java:211) at org.apache.cassandra.db.DeletionInfo.add(DeletionInfo.java:257) at org.apache.cassandra.db.AtomicBTreeColumns.addAllWithSizeDelta(AtomicBTreeColumns.java:217) at org.apache.cassandra.db.Memtable.put(Memtable.java:210) at org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1244) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:396) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:359) at org.apache.cassandra.db.Mutation.apply(Mutation.java:214) at org.apache.cassandra.service.StorageProxy$8.runMayThrow(StorageProxy.java:1086) at org.apache.cassandra.service.StorageProxy$LocalMutationRunnable$1.runMayThrow(StorageProxy.java:2275) at org.apache.cassandra.service.StorageProxy$HintRunnable.run(StorageProxy.java:2313) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) at java.lang.Thread.run(Thread.java:745) {code} was (Author: zhaoyan): 383 is BLOCKED. only one is running: {code} "SharedPool-Worker-27" #415 daemon prio=5 os_prio=0 tid=0x000000000242d800 nid=0x744e runnable [0x00007fdd8372c000] java.lang.Thread.State: RUNNABLE at java.util.Arrays.copyOf(Arrays.java:3284) at org.apache.cassandra.db.RangeTombstoneList.copy(RangeTombstoneList.java:116) at org.apache.cassandra.db.DeletionInfo.copy(DeletionInfo.java:104) at org.apache.cassandra.db.AtomicBTreeColumns.addAllWithSizeDelta(AtomicBTreeColumns.java:217) at org.apache.cassandra.db.Memtable.put(Memtable.java:210) at org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1244) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:396) at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:359) at org.apache.cassandra.db.Mutation.apply(Mutation.java:214) at org.apache.cassandra.db.MutationVerbHandler.doVerb(MutationVerbHandler.java:54) at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) at java.lang.Thread.run(Thread.java:745) {code} > All thread blocked and writes pending. > -------------------------------------- > > Key: CASSANDRA-13235 > URL: https://issues.apache.org/jira/browse/CASSANDRA-13235 > Project: Cassandra > Issue Type: Bug > Components: Core > Environment: jdk8 > cassandra 2.1.15 > Reporter: zhaoyan > > I found cassandra many pending MutationStage task > {code} > NFO [Service Thread] 2017-02-17 16:00:14,440 StatusLogger.java:51 - Pool > Name Active Pending Completed Blocked All Time > Blocked > INFO [Service Thread] 2017-02-17 16:00:14,440 StatusLogger.java:66 - > MutationStage 384 4553 4294213082 0 > 0 > INFO [Service Thread] 2017-02-17 16:00:14,441 StatusLogger.java:66 - > RequestResponseStage 0 0 2172612382 0 > 0 > INFO [Service Thread] 2017-02-17 16:00:14,441 StatusLogger.java:66 - > ReadRepairStage 0 0 5378852 0 > 0 > INFO [Service Thread] 2017-02-17 16:00:14,441 StatusLogger.java:66 - > CounterMutationStage 0 0 0 0 > 0 > INFO [Service Thread] 2017-02-17 16:00:14,441 StatusLogger.java:66 - > ReadStage 5 0 577242284 0 > 0 > INFO [Service Thread] 2017-02-17 16:00:14,441 StatusLogger.java:66 - > MiscStage 0 0 0 0 > 0 > INFO [Service Thread] 2017-02-17 16:00:14,441 StatusLogger.java:66 - > HintedHandoff 0 0 1480 0 > 0 > INFO [Service Thread] 2017-02-17 16:00:14,441 StatusLogger.java:66 - > GossipStage 0 0 9342250 0 > 0 > {code} > And I found there are many blocked thread with jstack > {code} > "SharedPool-Worker-28" #416 daemon prio=5 os_prio=0 tid=0x0000000001fb8000 > nid=0x7459 waiting for monitor entry [0x00007fdd83ca0000] > java.lang.Thread.State: BLOCKED (on object monitor) > at sun.misc.Unsafe.monitorEnter(Native Method) > at > org.apache.cassandra.utils.concurrent.Locks.monitorEnterUnsafe(Locks.java:46) > at > org.apache.cassandra.db.AtomicBTreeColumns.addAllWithSizeDelta(AtomicBTreeColumns.java:202) > at org.apache.cassandra.db.Memtable.put(Memtable.java:210) > at > org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1244) > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:396) > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:359) > at org.apache.cassandra.db.Mutation.apply(Mutation.java:214) > at > org.apache.cassandra.db.MutationVerbHandler.doVerb(MutationVerbHandler.java:54) > at > org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at > org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) > at java.lang.Thread.run(Thread.java:745) > {code} > To use "grep BLOCKED |wc -l", get Number is 384 -- This message was sent by Atlassian JIRA (v6.3.15#6346)