[ 
https://issues.apache.org/jira/browse/ARTEMIS-1286?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16115962#comment-16115962
 ] 

Phillip Jenkins commented on ARTEMIS-1286:
------------------------------------------

[~martyntaylor], checking to see if there's been any work on this yet. Any 
progress? Any estimate on when it might get to resolution?

> Server stops responding and throws OutOfDirectMemoryError when sending & 
> receiving lots of 2MB messages.
> --------------------------------------------------------------------------------------------------------
>
>                 Key: ARTEMIS-1286
>                 URL: https://issues.apache.org/jira/browse/ARTEMIS-1286
>             Project: ActiveMQ Artemis
>          Issue Type: Bug
>          Components: Broker, MQTT
>    Affects Versions: 2.1.0
>            Reporter: Phillip Jenkins
>            Assignee: Martyn Taylor
>            Priority: Blocker
>             Fix For: 2.3.0
>
>         Attachments: broker.xml, mqtt.zip
>
>
> Originally seen in v2.1 and present in v2.2.
> If you send and receive a lot of 2MB messages in short time thru Artemis via 
> Netty connector, the server throws the following OutOfDirectMemoryError 
> exception. The server stops responding and will not (any longer) accept 
> connections without throwing exceptions in an infinite loop.
> {code:java}
> 11:24:07,434 INFO  [org.apache.activemq.artemis] AMQ241001: HTTP Server 
> started at http://localhost:8161
> 11:24:07,434 INFO  [org.apache.activemq.artemis] AMQ241002: Artemis Jolokia 
> REST API available at http://localhost:8161/jolokia
> 11:58:35,991 WARN  [org.apache.activemq.artemis.core.server] AMQ222151: 
> removing consumer which did not handle a message, consumer=ServerConsumerImpl 
> [id=2229, filter=null, binding=LocalQueueBinding 
> [address=SOAP.S.PRN.ACBCAF6238234680, 
> queue=QueueImpl[name=ACBCAF6238234680.SOAP.S.PRN.ACBCAF6238234680, 
> postOffice=PostOfficeImpl 
> [server=ActiveMQServerImpl::serverUUID=afa7de73-67e7-11e7-a231-54ee7505882d], 
> temp=false]@46adc2a5, filter=FilterImpl [sfilterString=NOT ((AMQAddress = 
> 'activemq.management') OR (AMQAddress = 'activemq.notifications'))], 
> name=ACBCAF6238234680.SOAP.S.PRN.ACBCAF6238234680, 
> clusterName=ACBCAF6238234680.SOAP.S.PRN.ACBCAF6238234680afa7de73-67e7-11e7-a231-54ee7505882d]],
>  
> message=Reference[3551]:NON-RELIABLE:CoreMessage[messageID=3551,durable=false,userID=null,priority=0,
>  timestamp=0,expiration=0, durable=false, 
> address=SOAP.S.PRN.ACBCAF6238234680,properties=TypedProperties[mqtt.message.retain=false,mqtt.qos.level=0]]@1305748199:
>  io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 3729415 
> byte(s) of direct memory (used: 1070952692, max: 1073741824)
>       at 
> io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:585)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:539)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf.allocateDirect(UnpooledUnsafeNoCleanerDirectByteBuf.java:30)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf.allocateDirect(UnpooledByteBufAllocator.java:169)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.UnpooledUnsafeDirectByteBuf.<init>(UnpooledUnsafeDirectByteBuf.java:68)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf.<init>(UnpooledUnsafeNoCleanerDirectByteBuf.java:25)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf.<init>(UnpooledByteBufAllocator.java:164)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.UnpooledByteBufAllocator.newDirectBuffer(UnpooledByteBufAllocator.java:73)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:181)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:117)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.buffer.AbstractByteBuf.readBytes(AbstractByteBuf.java:828) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.buffer.WrappedByteBuf.readBytes(WrappedByteBuf.java:616) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> org.apache.activemq.artemis.core.buffers.impl.ChannelBufferWrapper.readBytes(ChannelBufferWrapper.java:315)
>  [artemis-commons-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.protocol.mqtt.MQTTPublishManager.sendServerMessage(MQTTPublishManager.java:277)
>  [artemis-mqtt-protocol-2.2.0-SNAPSHOT.jar:]
>       at 
> org.apache.activemq.artemis.core.protocol.mqtt.MQTTPublishManager.sendMessage(MQTTPublishManager.java:121)
>  [artemis-mqtt-protocol-2.2.0-SNAPSHOT.jar:]
>       at 
> org.apache.activemq.artemis.core.protocol.mqtt.MQTTSessionCallback.sendMessage(MQTTSessionCallback.java:52)
>  [artemis-mqtt-protocol-2.2.0-SNAPSHOT.jar:]
>       at 
> org.apache.activemq.artemis.core.server.impl.ServerConsumerImpl.deliverStandardMessage(ServerConsumerImpl.java:1078)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.server.impl.ServerConsumerImpl.proceedDeliver(ServerConsumerImpl.java:450)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.server.impl.QueueImpl.proceedDeliver(QueueImpl.java:2688)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.server.impl.QueueImpl.deliverDirect(QueueImpl.java:2673)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.server.impl.QueueImpl.addTail(QueueImpl.java:645)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl.addReferences(PostOfficeImpl.java:1297)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl.access$000(PostOfficeImpl.java:93)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl$1.done(PostOfficeImpl.java:1157)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.persistence.impl.journal.OperationContextImpl.executeOnCompletion(OperationContextImpl.java:181)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.persistence.impl.journal.OperationContextImpl.executeOnCompletion(OperationContextImpl.java:130)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.persistence.impl.journal.AbstractJournalStorageManager.afterCompleteOperations(AbstractJournalStorageManager.java:303)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl.processRoute(PostOfficeImpl.java:1149)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl.route(PostOfficeImpl.java:787)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl.route(PostOfficeImpl.java:689)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.server.impl.ServerSessionImpl.doSend(ServerSessionImpl.java:1674)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.server.impl.ServerSessionImpl.send(ServerSessionImpl.java:1351)
>  [artemis-server-2.2.0-SNAPSHOT.jar:2.2.0-SNAPSHOT]
>       at 
> org.apache.activemq.artemis.core.protocol.mqtt.MQTTPublishManager.sendInternal(MQTTPublishManager.java:166)
>  [artemis-mqtt-protocol-2.2.0-SNAPSHOT.jar:]
>       at 
> org.apache.activemq.artemis.core.protocol.mqtt.MQTTPublishManager.handleMessage(MQTTPublishManager.java:136)
>  [artemis-mqtt-protocol-2.2.0-SNAPSHOT.jar:]
>       at 
> org.apache.activemq.artemis.core.protocol.mqtt.MQTTProtocolHandler.handlePublish(MQTTProtocolHandler.java:204)
>  [artemis-mqtt-protocol-2.2.0-SNAPSHOT.jar:]
>       at 
> org.apache.activemq.artemis.core.protocol.mqtt.MQTTProtocolHandler.channelRead(MQTTProtocolHandler.java:118)
>  [artemis-mqtt-protocol-2.2.0-SNAPSHOT.jar:]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:624) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:559)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:476) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:438) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_72]
> {code}
> Subsequent attempts to connect with the server will fail and causes an 
> infinite loop throwing the following exception into the log until the server 
> is shut down.
> {code:java}
> 12:10:31,004 WARN  [io.netty.channel.DefaultChannelPipeline] An 
> exceptionCaught() event was fired, and it reached at the tail of the 
> pipeline. It usually means the last handler in the pipeline did not handle 
> the exception.: io.netty.util.internal.OutOfDirectMemoryError: failed to 
> allocate 16777216 byte(s) of direct memory (used: 1070952692, max: 1073741824)
>       at 
> io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:585)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:539)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:760) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:736) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:244) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.buffer.PoolArena.allocate(PoolArena.java:214) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.buffer.PoolArena.allocate(PoolArena.java:146) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:320)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:181)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:172)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.buffer.AbstractByteBufAllocator.ioBuffer(AbstractByteBufAllocator.java:133)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.DefaultMaxMessagesRecvByteBufAllocator$MaxMessageHandle.allocate(DefaultMaxMessagesRecvByteBufAllocator.java:80)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:122)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:624) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:559)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:476) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:438) 
> [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at 
> io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
>  [netty-all-4.1.9.Final.jar:4.1.9.Final]
>       at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_72]
> {code}



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Reply via email to