[ https://issues.apache.org/jira/browse/CASSANDRA-9762?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Philip Thompson updated CASSANDRA-9762: --------------------------------------- Reproduced In: 2.0.14 Fix Version/s: 2.0.x > Cassandra fails to start due to EOFException in > ColumnFamilyStore.removeUnfinishedCompactionLeftovers > ----------------------------------------------------------------------------------------------------- > > Key: CASSANDRA-9762 > URL: https://issues.apache.org/jira/browse/CASSANDRA-9762 > Project: Cassandra > Issue Type: Bug > Components: Core > Environment: SUSE 11.3 | C* 2.0.14 > Reporter: Jeremy Olmsted-Thompson > Fix For: 2.0.x > > > Cassandra failed to start with the following error: > {noformat} > ERROR [main] 2015-07-08 10:17:41,447 CassandraDaemon.java (line 584) > Exception encountered during startup > FSReadError in > /storage/core/loginsight/cidata/cassandra/data/logdb/spock_cluster_counts/logdb-spock_cluster_counts-tmp-jb-84218-Statistics.db > at > org.apache.cassandra.db.ColumnFamilyStore.removeUnfinishedCompactionLeftovers(ColumnFamilyStore.java:549) > at > org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:344) > at > org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:567) > at > org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:656) > Caused by: java.io.EOFException > at java.io.DataInputStream.readInt(Unknown Source) > at > org.apache.cassandra.utils.EstimatedHistogram$EstimatedHistogramSerializer.deserialize(EstimatedHistogram.java:334) > at > org.apache.cassandra.io.sstable.SSTableMetadata$SSTableMetadataSerializer.deserialize(SSTableMetadata.java:462) > at > org.apache.cassandra.io.sstable.SSTableMetadata$SSTableMetadataSerializer.deserialize(SSTableMetadata.java:448) > at > org.apache.cassandra.io.sstable.SSTableMetadata$SSTableMetadataSerializer.deserialize(SSTableMetadata.java:432) > at > org.apache.cassandra.db.ColumnFamilyStore.removeUnfinishedCompactionLeftovers(ColumnFamilyStore.java:545) > ... 3 more > {noformat} > Deleting the *tmp*.db files allowed startup to proceed normally. > About 5 minutes earlier during shutdown we see > {noformat} > ERROR [COMMIT-LOG-WRITER] 2015-07-08 10:12:10,765 CassandraDaemon.java (line > 258) Exception in thread Thread[COMMIT-LOG-WRITER,5,main] > java.util.concurrent.RejectedExecutionException: Task > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask@3585dd0f > rejected from > org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor@67741f3c[Terminated, > pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 5423] > at > java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(Unknown > Source) > at java.util.concurrent.ThreadPoolExecutor.reject(Unknown Source) > at > java.util.concurrent.ScheduledThreadPoolExecutor.delayedExecute(Unknown > Source) > at java.util.concurrent.ScheduledThreadPoolExecutor.schedule(Unknown > Source) > at java.util.concurrent.ScheduledThreadPoolExecutor.execute(Unknown > Source) > at > org.apache.cassandra.db.commitlog.CommitLogAllocator.flushOldestKeyspaces(CommitLogAllocator.java:333) > at > org.apache.cassandra.db.commitlog.CommitLogAllocator.fetchSegment(CommitLogAllocator.java:147) > at > org.apache.cassandra.db.commitlog.CommitLog.activateNextSegment(CommitLog.java:309) > at > org.apache.cassandra.db.commitlog.CommitLog.access$100(CommitLog.java:50) > at > org.apache.cassandra.db.commitlog.CommitLog$LogRecordAdder.run(CommitLog.java:360) > at > org.apache.cassandra.db.commitlog.PeriodicCommitLogExecutorService$1.runMayThrow(PeriodicCommitLogExecutorService.java:51) > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > at java.lang.Thread.run(Unknown Source) > {noformat} > and a couple > {noformat} > WARN [Native-Transport-Requests:30] 2015-07-08 10:12:11,364 Slf4JLogger.java > (line 76) An exception was thrown by an exception handler. > java.util.concurrent.RejectedExecutionException: Worker has already been > shutdown > at > org.jboss.netty.channel.socket.nio.AbstractNioSelector.registerTask(AbstractNioSelector.java:115) > at > org.jboss.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:73) > at > org.jboss.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:36) > at > org.jboss.netty.channel.socket.nio.AbstractNioWorker.executeInIoThread(AbstractNioWorker.java:57) > at > org.jboss.netty.channel.socket.nio.NioWorker.executeInIoThread(NioWorker.java:36) > at > org.jboss.netty.channel.socket.nio.AbstractNioChannelSink.execute(AbstractNioChannelSink.java:34) > at > org.jboss.netty.channel.Channels.fireExceptionCaughtLater(Channels.java:496) > at > org.jboss.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:46) > at org.jboss.netty.channel.Channels.write(Channels.java:725) > at > org.jboss.netty.handler.codec.oneone.OneToOneEncoder.doEncode(OneToOneEncoder.java:71) > at > org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:59) > at org.jboss.netty.channel.Channels.write(Channels.java:725) > at > org.jboss.netty.handler.codec.oneone.OneToOneEncoder.doEncode(OneToOneEncoder.java:71) > at > org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:59) > at org.jboss.netty.channel.Channels.write(Channels.java:725) > at > org.jboss.netty.handler.codec.oneone.OneToOneEncoder.doEncode(OneToOneEncoder.java:71) > at > org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:59) > at > org.jboss.netty.handler.execution.ExecutionHandler.handleDownstream(ExecutionHandler.java:186) > at org.jboss.netty.channel.Channels.write(Channels.java:704) > at org.jboss.netty.channel.Channels.write(Channels.java:671) > at > org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248) > at > org.apache.cassandra.transport.Message$Dispatcher.messageReceived(Message.java:328) > at > org.jboss.netty.handler.execution.ChannelUpstreamEventRunnable.doRun(ChannelUpstreamEventRunnable.java:43) > at > org.jboss.netty.handler.execution.ChannelEventRunnable.run(ChannelEventRunnable.java:67) > at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) > at java.lang.Thread.run(Unknown Source) > {noformat} > There were no other errors within the last hour prior to this event. -- This message was sent by Atlassian JIRA (v6.3.4#6332)