[
https://issues.apache.org/jira/browse/CASSANDRA-10689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15000901#comment-15000901
]
mlowicki commented on CASSANDRA-10689:
--------------------------------------
After upgrade from 2.1.9 to 2.1.11 two days ago I'm getting lots of:
{code}
WARN [SharedPool-Worker-28] 2015-11-11 19:01:22,409
AbstractTracingAwareExecutorService.java:169 - Uncaught exception on thread
Thread[SharedPool-Worker-28,5,main]: {}
org.apache.cassandra.io.sstable.CorruptSSTableException:
org.apache.cassandra.io.compress.CorruptBlockException:
(/var/lib/cassandra/data2/sync/entity2-e24b5040199b11e5a30f75bb514ae072/sync-entity2-ka-392603-Data.db):
corruption detected, chunk at 11612338 of length 156219476.
at
org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer(CompressedRandomAccessReader.java:85)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:310)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.io.util.PoolingSegmentedFile.getSegment(PoolingSegmentedFile.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.io.sstable.SSTableReader.getFileDataInput(SSTableReader.java:1894)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.columniterator.IndexedSliceReader.setToRowStart(IndexedSliceReader.java:107)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.columniterator.IndexedSliceReader.<init>(IndexedSliceReader.java:83)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:65)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:42)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:246)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:62)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:270)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:62)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1994)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1837)
~[apache-cassandra-2.1.11.jar:2.1.11]
at org.apache.cassandra.db.Keyspace.getRow(Keyspace.java:353)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:85)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:47)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
~[na:1.7.0_80]
at
org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
~[apache-cassandra-2.1.11.jar:2.1.11]
at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
[apache-cassandra-2.1.11.jar:2.1.11]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: org.apache.cassandra.io.compress.CorruptBlockException:
(/var/lib/cassandra/data2/sync/entity2-e24b5040199b11e5a30f75bb514ae072/sync-entity2-ka-392603-Data.db):
corruption detected, chunk at 11612338 of length 156219476.
at
org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(CompressedRandomAccessReader.java:116)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer(CompressedRandomAccessReader.java:81)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 21 common frames omitted
Caused by: java.io.IOException: Compressed lengths mismatch
at
org.apache.cassandra.io.compress.LZ4Compressor.uncompress(LZ4Compressor.java:98)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(CompressedRandomAccessReader.java:112)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 22 common frames omitted
{code}
On 3 out of 7 nodes in one data center.
> java.lang.OutOfMemoryError: Direct buffer memory
> ------------------------------------------------
>
> Key: CASSANDRA-10689
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10689
> Project: Cassandra
> Issue Type: Bug
> Reporter: mlowicki
>
> {code}
> ERROR [SharedPool-Worker-63] 2015-11-11 17:53:16,161
> JVMStabilityInspector.java:117 - JVM state determined to be unstable.
> Exiting forcefully due to:
> java.lang.OutOfMemoryError: Direct buffer memory
> at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.7.0_80]
> at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
> ~[na:1.7.0_80]
> at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306)
> ~[na:1.7.0_80]
> at sun.nio.ch.Util.getTemporaryDirectBuffer(Util.java:174)
> ~[na:1.7.0_80]
> at sun.nio.ch.IOUtil.read(IOUtil.java:195) ~[na:1.7.0_80]
> at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:149)
> ~[na:1.7.0_80]
> at
> org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(CompressedRandomAccessReader.java:104)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer(CompressedRandomAccessReader.java:81)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:310)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.util.PoolingSegmentedFile.getSegment(PoolingSegmentedFile.java:64)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.sstable.SSTableReader.getFileDataInput(SSTableReader.java:1894)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.IndexedSliceReader.setToRowStart(IndexedSliceReader.java:107)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.IndexedSliceReader.<init>(IndexedSliceReader.java:83)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:65)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:42)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:246)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:62)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:270)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:62)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1994)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1837)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at org.apache.cassandra.db.Keyspace.getRow(Keyspace.java:353)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:85)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:47)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> ~[na:1.7.0_80]
> at
> org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
> [apache-cassandra-2.1.11.jar:2.1.11]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
