Experiencing a recurring issue with our activeMQ broker, where it will stop servicing new connections (any new connection from the client side hangs on establishing a connection). Thread dump attached, and a small selection of the dump pasted below. Our configuration is also listed below (and attached).
<beans xmlns="http://www.springframework.org/schema/beans"; xmlns:amq="http://activemq.apache.org/schema/core"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd";> <bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> <property name="locations"> <value>file:${activemq.conf}/credentials.properties</value> </property> </bean> <broker xmlns="http://activemq.apache.org/schema/core"; brokerName="main" dataDirectory="${activemq.data}" offlineDurableSubscriberTimeout="86400000" offlineDurableSubscriberTaskSchedule="21600000"> <destinationPolicy> <policyMap> <policyEntries> <policyEntry topic=">" producerFlowControl="false" advisoryForSlowConsumers="true" advisoryForFastProducers="true" advisoryWhenFull="true" memoryLimit="4096mb"> <dispatchPolicy> <simpleDispatchPolicy/> </dispatchPolicy> <pendingMessageLimitStrategy> <constantPendingMessageLimitStrategy limit="10000"/> </pendingMessageLimitStrategy> <messageEvictionStrategy> <oldestMessageEvictionStrategy/> </messageEvictionStrategy> </policyEntry> <policyEntry queue=">" producerFlowControl="false" optimizedDispatch="true" useCache="false" memoryLimit="4096mb"> <pendingMessageLimitStrategy> <constantPendingMessageLimitStrategy limit="10000"/> </pendingMessageLimitStrategy> <messageEvictionStrategy> <oldestMessageEvictionStrategy/> </messageEvictionStrategy> </policyEntry> <policyEntry topic=">" tempTopic="true" producerFlowControl="false" advisoryForSlowConsumers="true" advisoryForFastProducers="true" advisoryWhenFull="true" memoryLimit="4096mb"> </policyEntry> <policyEntry queue=">" tempQueue="true" producerFlowControl="false" optimizedDispatch="true" useCache="false" advisoryForSlowConsumers="true" advisoryForFastProducers="true" advisoryWhenFull="true" memoryLimit="4096mb"> </policyEntry> </policyEntries> </policyMap> </destinationPolicy> <managementContext> <managementContext createConnector="true" connectorPort="1099" rmiServerPort="1199"/> </managementContext> <persistenceAdapter> <kahaDB directory="${activemq.data}/kahadb-main"/> </persistenceAdapter> <systemUsage> <systemUsage> <memoryUsage> <memoryUsage limit="8 gb"/> </memoryUsage> <storeUsage> <storeUsage limit="20 gb"/> </storeUsage> <tempUsage> <tempUsage limit="50 gb"/> </tempUsage> </systemUsage> </systemUsage> <transportConnectors> <transportConnector name="main-openwire" uri="tcp://0.0.0.0:61616"/> </transportConnectors> <shutdownHooks> <bean xmlns="http://www.springframework.org/schema/beans"; class="org.apache.activemq.hooks.SpringContextHook" /> </shutdownHooks> </broker> <import resource="jetty-main.xml"/> </beans> Selection of thread dumps: "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" - Thread t@1545 java.lang.Thread.State: RUNNABLE at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(Unknown Source) at java.net.SocketOutputStream.write(Unknown Source) at org.apache.activemq.transport.tcp.TcpBufferedOutputStream.flush(TcpBufferedOutputStream.java:115) at java.io.DataOutputStream.flush(Unknown Source) at org.apache.activemq.transport.tcp.TcpTransport.oneway(TcpTransport.java:176) at org.apache.activemq.transport.AbstractInactivityMonitor.doOnewaySend(AbstractInactivityMonitor.java:304) at org.apache.activemq.transport.AbstractInactivityMonitor.oneway(AbstractInactivityMonitor.java:286) at org.apache.activemq.transport.TransportFilter.oneway(TransportFilter.java:85) at org.apache.activemq.transport.WireFormatNegotiator.oneway(WireFormatNegotiator.java:104) at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) at org.apache.activemq.broker.TransportConnection.dispatch(TransportConnection.java:1339) at org.apache.activemq.broker.TransportConnection.processDispatch(TransportConnection.java:858) at org.apache.activemq.broker.TransportConnection.dispatchSync(TransportConnection.java:818) at org.apache.activemq.broker.region.TopicSubscription.dispatch(TopicSubscription.java:611) at org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscription.java:112) at org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch(SimpleDispatchPolicy.java:48) at org.apache.activemq.broker.region.Topic.dispatch(Topic.java:705) at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:504) - locked <6f8e78f0> (a org.apache.activemq.broker.region.Topic) at org.apache.activemq.broker.region.Topic.send(Topic.java:440) at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394) at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442) at org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511) at org.apache.activemq.advisory.AdvisoryBroker.addDestinationInfo(AdvisoryBroker.java:206) at org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) at org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) at org.apache.activemq.broker.MutableBrokerFilter.addDestinationInfo(MutableBrokerFilter.java:262) at org.apache.activemq.broker.TransportConnection.processAddDestination(TransportConnection.java:495) at org.apache.activemq.command.DestinationInfo.visit(DestinationInfo.java:122) at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:292) at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:149) at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50) at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113) at org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:270) at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83) at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214) at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <6c833519> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ActiveMQ Transport: tcp:///10.0.1.229:51551@61616" - Thread t@1544 java.lang.Thread.State: BLOCKED at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:459) - waiting to lock <6f8e78f0> (a org.apache.activemq.broker.region.Topic) owned by "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" t@1545 at org.apache.activemq.broker.region.Topic.send(Topic.java:440) at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394) at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442) at org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511) at org.apache.activemq.advisory.AdvisoryBroker.addDestinationInfo(AdvisoryBroker.java:206) at org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) at org.apache.activemq.broker.BrokerFilter.addDestinationInfo(BrokerFilter.java:257) at org.apache.activemq.broker.MutableBrokerFilter.addDestinationInfo(MutableBrokerFilter.java:262) at org.apache.activemq.broker.TransportConnection.processAddDestination(TransportConnection.java:495) at org.apache.activemq.command.DestinationInfo.visit(DestinationInfo.java:122) at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:292) at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:149) at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50) at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113) at org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:270) at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83) at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214) at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "ActiveMQ BrokerService[main] Task-374" - Thread t@1906 java.lang.Thread.State: BLOCKED at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:459) - waiting to lock <6f8e78f0> (a org.apache.activemq.broker.region.Topic) owned by "ActiveMQ Transport: tcp:///10.0.1.229:51552@61616" t@1545 at org.apache.activemq.broker.region.Topic.send(Topic.java:440) at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:394) at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:442) at org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:283) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:585) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:516) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:511) at org.apache.activemq.advisory.AdvisoryBroker.removeDestination(AdvisoryBroker.java:221) at org.apache.activemq.broker.BrokerFilter.removeDestination(BrokerFilter.java:172) at org.apache.activemq.broker.BrokerFilter.removeDestination(BrokerFilter.java:172) at org.apache.activemq.broker.MutableBrokerFilter.removeDestination(MutableBrokerFilter.java:177) at org.apache.activemq.broker.TransportConnection.processRemoveConnection(TransportConnection.java:779) - locked <6f39a3be> (a org.apache.activemq.broker.jmx.ManagedTransportConnection) at org.apache.activemq.broker.TransportConnection.doStop(TransportConnection.java:1107) at org.apache.activemq.broker.TransportConnection$4.run(TransportConnection.java:1037) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <312d47fe> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - locked <1556e994> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ActiveMQ BrokerService[main] Task-396" - Thread t@1959 java.lang.Thread.State: TIMED_WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <6eb8ca36> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(Unknown Source) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(Unknown Source) at java.util.concurrent.SynchronousQueue.poll(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "ActiveMQ Transport: tcp:///10.0.1.202:59370@61616" - Thread t@1498 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50) at org.apache.activemq.transport.tcp.TcpTransport$2.fill(TcpTransport.java:604) at org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:58) at org.apache.activemq.transport.tcp.TcpTransport$2.read(TcpTransport.java:589) at java.io.DataInputStream.readInt(Unknown Source) at org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:275) at org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:221) at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:213) at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None threaddump-1393608740161.tdump <http://activemq.2283324.n4.nabble.com/file/n4678423/threaddump-1393608740161.tdump> activemq.xml <http://activemq.2283324.n4.nabble.com/file/n4678423/activemq.xml> -- View this message in context: http://activemq.2283324.n4.nabble.com/Deadlock-on-broker-thread-dumps-included-tp4678423.html Sent from the ActiveMQ - User mailing list archive at Nabble.com.
