[ https://issues.apache.org/jira/browse/CASSANDRA-4707?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13484143#comment-13484143 ]
Ivan Velykorodnyy commented on CASSANDRA-4707: ---------------------------------------------- Looks like a similar error here: # TimedOutException on the client # EOFException on the single cassandra 1.1.5 node: {code} ERROR [ReadStage:3028] 2012-10-25 06:40:37,406 AbstractCassandraDaemon.java (line 135) Exception in thread Thread[ReadStage:3028,5,main] java.io.IOError: java.io.EOFException at org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:66) at org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:67) at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:79) at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:256) at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:64) at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1345) at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207) at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1142) at org.apache.cassandra.db.Table.getRow(Table.java:378) at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:69) at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:819) at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1253) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) Caused by: java.io.EOFException at java.io.RandomAccessFile.readFully(RandomAccessFile.java:399) at java.io.RandomAccessFile.readFully(RandomAccessFile.java:377) at org.apache.cassandra.io.util.RandomAccessReader.readBytes(RandomAccessReader.java:324) at org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:398) at org.apache.cassandra.utils.ByteBufferUtil.readWithShortLength(ByteBufferUtil.java:380) at org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:56) ... 14 more {code} # node restart resolved the issue There was a scheduled test processing 500 msgs (some cql3 selects+single insert per a message) every 15 mins. Test failed in the middle and all consecutive runs failed as well. Cassandra node restart fixed test. > EOFException during HH delivery > ------------------------------- > > Key: CASSANDRA-4707 > URL: https://issues.apache.org/jira/browse/CASSANDRA-4707 > Project: Cassandra > Issue Type: Bug > Affects Versions: 1.1.5 > Environment: windows > Reporter: Radim Kolar > > i have got following error during HH every replay. I did nodetool scrub on > system.hintedhandoff and sstable is not corrupted, it must have invalid data > inside. > INFO [HintedHandoff:10] 2012-09-23 20:26:21,988 HintedHandOffManager.java > (line 294) Started hinted handoff for token: > 138224439286689469893643387845607487221 with IP: /10.0.0.9 > ERROR [HintedHandoff:10] 2012-09-23 20:26:21,988 AbstractCassandraDaemon.java > (line 135) Exception in thread Thread[HintedHandoff:10,1,main] > java.io.IOError: java.io.EOFException > at > org.apache.cassandra.io.util.ColumnIterator.deserializeNext(ColumnSortedMap.java:259) > at > org.apache.cassandra.io.util.ColumnIterator.next(ColumnSortedMap.java:275) > at > org.apache.cassandra.io.util.ColumnIterator.next(ColumnSortedMap.java:232) > at edu.stanford.ppl.concurrent.SnapTreeMap.<init>(SnapTreeMap.java:453) > at > org.apache.cassandra.db.AtomicSortedColumns$Holder.<init>(AtomicSortedColumns.java:311) > at > org.apache.cassandra.db.AtomicSortedColumns.<init>(AtomicSortedColumns.java:77) > at > org.apache.cassandra.db.AtomicSortedColumns.<init>(AtomicSortedColumns.java:48) > at > org.apache.cassandra.db.AtomicSortedColumns$1.fromSorted(AtomicSortedColumns.java:61) > at > org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:399) > at > org.apache.cassandra.db.ColumnFamilySerializer.deserializeColumns(ColumnFamilySerializer.java:144) > at > org.apache.cassandra.db.ColumnFamilySerializer.deserializeColumns(ColumnFamilySerializer.java:136) > at > org.apache.cassandra.db.ColumnFamilySerializer.deserialize(ColumnFamilySerializer.java:129) > at > org.apache.cassandra.db.RowMutation$RowMutationSerializer.deserialize(RowMutation.java:439) > at > org.apache.cassandra.db.RowMutation$RowMutationSerializer.deserialize(RowMutation.java:447) > at > org.apache.cassandra.db.HintedHandOffManager.deliverHintsToEndpointInternal(HintedHandOffManager.java:353) > at > org.apache.cassandra.db.HintedHandOffManager.deliverHintsToEndpoint(HintedHandOffManager.java:256) > at > org.apache.cassandra.db.HintedHandOffManager.access$300(HintedHandOffManager.java:84) > at > org.apache.cassandra.db.HintedHandOffManager$3.runMayThrow(HintedHandOffManager.java:436) > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:30) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) > at java.lang.Thread.run(Thread.java:722) > Caused by: java.io.EOFException > at java.io.DataInputStream.readFully(DataInputStream.java:197) > at java.io.DataInputStream.readFully(DataInputStream.java:169) > at > org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:401) > at > org.apache.cassandra.utils.ByteBufferUtil.readWithShortLength(ByteBufferUtil.java:380) > at > org.apache.cassandra.db.ColumnSerializer.deserialize(ColumnSerializer.java:88) > at > org.apache.cassandra.io.util.ColumnIterator.deserializeNext(ColumnSortedMap.java:255) > ... 21 more -- This message is automatically generated by JIRA. If you think it was sent incorrectly, please contact your JIRA administrators For more information on JIRA, see: http://www.atlassian.com/software/jira