After upgrading my cluster from 0.5.1 to the 0.6 branch (commit 1206bcf in
git). I am seeing lots of IO errors in the log output.
Two questions:
1. Is this a sign that I have corrupt data? Is there some way for me to
recover it or at the very least remove the bad data?
2. If this is an issue with 0.6 can I safely downgrading back to 0.5.1?
There are three different error's that I am seeing.
Exception 1:
ERROR 00:21:23,419 Error in ThreadPoolExecutor
java.lang.RuntimeException: java.io.IOException: Corrupt (negative) value
length encountered
at
org.apache.cassandra.db.filter.SSTableSliceIterator$ColumnGroupReader.pollColumn(SSTableSliceIterator.java:159)
at
org.apache.cassandra.db.filter.SSTableSliceIterator.computeNext(SSTableSliceIterator.java:92)
at
org.apache.cassandra.db.filter.SSTableSliceIterator.computeNext(SSTableSliceIterator.java:39)
at
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:135)
at
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:130)
at
org.apache.commons.collections.iterators.CollatingIterator.anyHasNext(CollatingIterator.java:364)
at
org.apache.commons.collections.iterators.CollatingIterator.hasNext(CollatingIterator.java:217)
at
org.apache.cassandra.utils.ReducingIterator.computeNext(ReducingIterator.java:55)
at
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:135)
at
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:130)
at
org.apache.cassandra.db.filter.SliceQueryFilter.collectReducedColumns(SliceQueryFilter.java:109)
at
org.apache.cassandra.db.filter.QueryFilter.collectCollatedColumns(QueryFilter.java:112)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:869)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:770)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:739)
at org.apache.cassandra.db.Table.getRow(Table.java:381)
at
org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:59)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:70)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.io.IOException: Corrupt (negative) value length encountered
at
org.apache.cassandra.db.ColumnSerializer.deserialize(ColumnSerializer.java:82)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:335)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:284)
at
org.apache.cassandra.db.filter.SSTableSliceIterator$ColumnGroupReader.getNextBlock(SSTableSliceIterator.java:194)
at
org.apache.cassandra.db.filter.SSTableSliceIterator$ColumnGroupReader.pollColumn(SSTableSliceIterator.java:154)
... 21 more
Exception 2:
ERROR 00:30:27,752 Fatal exception in thread
Thread[ROW-READ-STAGE:85,5,main]
java.lang.RuntimeException: java.io.IOException: Corrupt (negative) value
length encountered
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:106)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.io.IOException: Corrupt (negative) value length encountered
at
org.apache.cassandra.db.ColumnSerializer.deserialize(ColumnSerializer.java:82)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:335)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:284)
at
org.apache.cassandra.db.filter.SSTableNamesIterator.<init>(SSTableNamesIterator.java:104)
at
org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:69)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:859)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:770)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:739)
at org.apache.cassandra.db.Table.getRow(Table.java:381)
at
org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:56)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:70)
... 4 more
Exception 3:
ERROR 00:08:27,909 Error in ThreadPoolExecutor
java.lang.RuntimeException: java.io.EOFException
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:106)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.io.EOFException
at
org.apache.cassandra.io.util.MappedFileDataInput.readFully(MappedFileDataInput.java:261)
at
org.apache.cassandra.io.util.MappedFileDataInput.readFully(MappedFileDataInput.java:223)
at
org.apache.cassandra.db.ColumnSerializer.deserialize(ColumnSerializer.java:87)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:335)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:284)
at
org.apache.cassandra.db.filter.SSTableNamesIterator.<init>(SSTableNamesIterator.java:104)
at
org.apache.cassandra.db.filter.NamesQueryFilter.getSSTableColumnIterator(NamesQueryFilter.java:69)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:859)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:770)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:739)
at org.apache.cassandra.db.Table.getRow(Table.java:381)
at
org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:56)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:70)
... 4 more
