[
https://issues.apache.org/jira/browse/CASSANDRA-10689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15180691#comment-15180691
]
Digant Modha edited comment on CASSANDRA-10689 at 3/4/16 10:59 PM:
-------------------------------------------------------------------
I also have a similar issue:
Top shows process is using 19g memory. The memory usage grows until node dies.
It is started with -Xmx8g -Xms8g -Xmn2g.
java.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.8.0_60]
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
~[na:1.8.0_60]
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
~[na:1.8.0_60]
at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:580)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:456)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.handler.ssl.SslHandler.flush(SslHandler.java:432)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
org.apache.cassandra.transport.Message$Dispatcher$Flusher.run(Message.java:389)
~[apache-cassandra-2.1.12.jar:2.1.12]
at
io.netty.util.concurrent.PromiseTask$RunnableAdapter.call(PromiseTask.java:38)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:123)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:380)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:268)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:116)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_60]
was (Author: dmodha):
I also have a similar issue:
ava.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.8.0_60]
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
~[na:1.8.0_60]
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
~[na:1.8.0_60]
at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:580)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:456)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.handler.ssl.SslHandler.flush(SslHandler.java:432)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:688)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:669)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
org.apache.cassandra.transport.Message$Dispatcher$Flusher.run(Message.java:389)
~[apache-cassandra-2.1.12.jar:2.1.12]
at
io.netty.util.concurrent.PromiseTask$RunnableAdapter.call(PromiseTask.java:38)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:123)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:380)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:268)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:116)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
~[netty-all-4.0.23.Final.jar:4.0.23.Final]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_60]
> java.lang.OutOfMemoryError: Direct buffer memory
> ------------------------------------------------
>
> Key: CASSANDRA-10689
> URL: https://issues.apache.org/jira/browse/CASSANDRA-10689
> Project: Cassandra
> Issue Type: Bug
> Reporter: mlowicki
>
> {code}
> ERROR [SharedPool-Worker-63] 2015-11-11 17:53:16,161
> JVMStabilityInspector.java:117 - JVM state determined to be unstable.
> Exiting forcefully due to:
> java.lang.OutOfMemoryError: Direct buffer memory
> at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.7.0_80]
> at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
> ~[na:1.7.0_80]
> at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306)
> ~[na:1.7.0_80]
> at sun.nio.ch.Util.getTemporaryDirectBuffer(Util.java:174)
> ~[na:1.7.0_80]
> at sun.nio.ch.IOUtil.read(IOUtil.java:195) ~[na:1.7.0_80]
> at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:149)
> ~[na:1.7.0_80]
> at
> org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(CompressedRandomAccessReader.java:104)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer(CompressedRandomAccessReader.java:81)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:310)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.util.PoolingSegmentedFile.getSegment(PoolingSegmentedFile.java:64)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.io.sstable.SSTableReader.getFileDataInput(SSTableReader.java:1894)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.IndexedSliceReader.setToRowStart(IndexedSliceReader.java:107)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.IndexedSliceReader.<init>(IndexedSliceReader.java:83)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:65)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:42)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:246)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:62)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:270)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:62)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1994)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1837)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at org.apache.cassandra.db.Keyspace.getRow(Keyspace.java:353)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:85)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:47)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> ~[na:1.7.0_80]
> at
> org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164)
> ~[apache-cassandra-2.1.11.jar:2.1.11]
> at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
> [apache-cassandra-2.1.11.jar:2.1.11]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
