Something has shutdown the mutation stage thread pool. This happens during drain or decommission / move.
Restart the service and it should be ok. if it happens again without anyone running something like drain, decommission or move let us know. Cheers ----------------- Aaron Morton Freelance Cassandra Developer @aaronmorton http://www.thelastpickle.com On 22 Jul 2011, at 19:41, lebron james wrote: > ERROR [pool-2-thread-3] 2011-07-22 10:34:59,102 Cassandra.java (line 3294) > Internal error processing insert > java.util.concurrent.RejectedExecutionException: ThreadPoolExecutor has shut > down > at > org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor$1.rejectedExecution(DebuggableThreadPoolExecutor.java:73) > at java.util.concurrent.ThreadPoolExecutor.reject(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor.execute(Unknown Source) > at > org.apache.cassandra.service.StorageProxy.insertLocal(StorageProxy.java:360) > at > org.apache.cassandra.service.StorageProxy.sendToHintedEndpoints(StorageProxy.java:241) > at org.apache.cassandra.service.StorageProxy.access$000(StorageProxy.java:62) > at org.apache.cassandra.service.StorageProxy$1.apply(StorageProxy.java:99) > at > org.apache.cassandra.service.StorageProxy.performWrite(StorageProxy.java:210) > at org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:154) > at > org.apache.cassandra.thrift.CassandraServer.doInsert(CassandraServer.java:560) > at > org.apache.cassandra.thrift.CassandraServer.internal_insert(CassandraServer.java:436) > at > org.apache.cassandra.thrift.CassandraServer.insert(CassandraServer.java:444) > at > org.apache.cassandra.thrift.Cassandra$Processor$insert.process(Cassandra.java:3286) > at > org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:2889) > at > org.apache.cassandra.thrift.CustomTThreadPoolServer$WorkerProcess.run(CustomTThreadPoolServer.java:187) > at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.lang.Thread.run(Unknown Source) > ERROR [pool-2-thread-6] 2011-07-22 10:34:59,102 Cassandra.java (line 3294) > Internal error processing insert > java.util.concurrent.RejectedExecutionException: ThreadPoolExecutor has shut > down > at > org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor$1.rejectedExecution(DebuggableThreadPoolExecutor.java:73) > at java.util.concurrent.ThreadPoolExecutor.reject(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor.execute(Unknown Source) > at > org.apache.cassandra.service.StorageProxy.insertLocal(StorageProxy.java:360) > at > org.apache.cassandra.service.StorageProxy.sendToHintedEndpoints(StorageProxy.java:241) > at org.apache.cassandra.service.StorageProxy.access$000(StorageProxy.java:62) > at org.apache.cassandra.service.StorageProxy$1.apply(StorageProxy.java:99) > at > org.apache.cassandra.service.StorageProxy.performWrite(StorageProxy.java:210) > at org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:154) > at > org.apache.cassandra.thrift.CassandraServer.doInsert(CassandraServer.java:560) > at > org.apache.cassandra.thrift.CassandraServer.internal_insert(CassandraServer.java:436) > at > org.apache.cassandra.thrift.CassandraServer.insert(CassandraServer.java:444) > at > org.apache.cassandra.thrift.Cassandra$Processor$insert.process(Cassandra.java:3286) > at > org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:2889) > at > org.apache.cassandra.thrift.CustomTThreadPoolServer$WorkerProcess.run(CustomTThreadPoolServer.java:187) > at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.lang.Thread.run(Unknown Source) > ERROR [pool-2-thread-3] 2011-07-22 10:34:59,102 Cassandra.java (line 3294) > Internal error processing insert > java.util.concurrent.RejectedExecutionException: ThreadPoolExecutor has shut > down > at > org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor$1.rejectedExecution(DebuggableThreadPoolExecutor.java:73) > at java.util.concurrent.ThreadPoolExecutor.reject(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor.execute(Unknown Source) > at > org.apache.cassandra.service.StorageProxy.insertLocal(StorageProxy.java:360) > at > org.apache.cassandra.service.StorageProxy.sendToHintedEndpoints(StorageProxy.java:241) > at org.apache.cassandra.service.StorageProxy.access$000(StorageProxy.java:62) > at org.apache.cassandra.service.StorageProxy$1.apply(StorageProxy.java:99) > at > org.apache.cassandra.service.StorageProxy.performWrite(StorageProxy.java:210) > at org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:154) > at > org.apache.cassandra.thrift.CassandraServer.doInsert(CassandraServer.java:560) > at > org.apache.cassandra.thrift.CassandraServer.internal_insert(CassandraServer.java:436) > at > org.apache.cassandra.thrift.CassandraServer.insert(CassandraServer.java:444) > at > org.apache.cassandra.thrift.Cassandra$Processor$insert.process(Cassandra.java:3286) > at > org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:2889) > at > org.apache.cassandra.thrift.CustomTThreadPoolServer$WorkerProcess.run(CustomTThreadPoolServer.java:187) > at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.lang.Thread.run(Unknown Source) > > > >
