Can you disabled the auto removal of the inactive destination feature?
seems deadlock is present...
At 2013-05-30 17:26:00,"Tlholoe, Peter" <ptlho...@fnb.co.za> wrote:
>Good Day,
>
>We currently have a problem with our ActiveMQ periodically hanging in PROD,
>and I did take a thread dumps for a period of 10 minutes, and there are a
>couple of threads that are getting blocked. Please see below all the threads
>that are blocked. Can you please help ascertain what might be happening here.
>
>
>
>Can you please help me diagnose what might be happening here
>
>Thread One
>"ActiveMQ Transport: tcp:///{IP}:40621" daemon prio=10 tid=0x000000005ddb1800
>nid=0x3d4d waiting for monitor entry [0x00002b3c87634000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:430)
> - waiting to lock
> <0x000000076037a788<../sequence/threadId-0x000000005ddee800_shrink-true.html#0x000000076037a788_0>>
> (a org.apache.activemq.broker.region.Topic)
> at org.apache.activemq.broker.region.Topic.send(Topic.java:411)
> at
> org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
> at
> org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:518)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at
> org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
> at
> org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
> at
> org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
> at
> org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:462)
> at
> org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:677)
> at
> org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:311)
> at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:185)
> at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:228)
> - locked < 0x00000007d054b100> (a
> org.apache.activemq.transport.InactivityMonitor$1)
> at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
> at java.lang.Thread.run(Thread.java:662)
>
>Thread Two
>"ActiveMQ Broker[localhost] Scheduler" daemon prio=10 tid=0x00002b3c780d6800
>nid=0x1403 waiting for monitor entry [0x00002b3c7cce8000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.activemq.broker.region.RegionBroker.purgeInactiveDestinations(RegionBroker.java:920)
> - waiting to lock
> <0x00000007603866f8<../sequence/threadId-0x000000005ded2000_shrink-true.html#0x00000007603866f8_0>>
> (a org.apache.activemq.broker.region.RegionBroker$1)
> at
> org.apache.activemq.broker.region.RegionBroker$1.run(RegionBroker.java:110)
> at
> org.apache.activemq.thread.SchedulerTimerTask.run(SchedulerTimerTask.java:33)
> at java.util.TimerThread.mainLoop(Timer.java:512)
> at java.util.TimerThread.run(Timer.java:462)
>
>What I saw as well
>"ActiveMQ Transport: tcp:///{IP}:45395" daemon prio=10 tid=0x000000005ded2000
>nid=0xe22 waiting on condition [0x00002b3c84a07000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x000000076037c188> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:811)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:941)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1261)
> at
> java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:594)
> at
> org.apache.activemq.store.jdbc.adapter.DefaultJDBCAdapter.doGetDurableSubscriberMessageCount(DefaultJDBCAdapter.java:575)
> at
> org.apache.activemq.store.jdbc.JDBCTopicMessageStore.getMessageCount(JDBCTopicMessageStore.java:223)
> at
> org.apache.activemq.store.ProxyTopicMessageStore.getMessageCount(ProxyTopicMessageStore.java:116)
> at
> org.apache.activemq.broker.region.cursors.TopicStorePrefetch.getStoreSize(TopicStorePrefetch.java:77)
> - locked < 0x00000007603869d0> (a
> org.apache.activemq.broker.region.cursors.TopicStorePrefetch)
> at
> org.apache.activemq.broker.region.cursors.AbstractStoreCursor.start(AbstractStoreCursor.java:59)
> - locked < 0x00000007603869d0> (a
> org.apache.activemq.broker.region.cursors.TopicStorePrefetch)
> at
> org.apache.activemq.broker.region.cursors.StoreDurableSubscriberCursor.start(StoreDurableSubscriberCursor.java:81)
> - locked < 0x00000007603868e8> (a
> org.apache.activemq.broker.region.cursors.StoreDurableSubscriberCursor)
> at
> org.apache.activemq.broker.region.DurableTopicSubscription.activate(DurableTopicSubscription.java:137)
> - locked < 0x00000007603868e8> (a
> org.apache.activemq.broker.region.cursors.StoreDurableSubscriberCursor)
> at
> org.apache.activemq.broker.region.TopicRegion.addConsumer(TopicRegion.java:108)
> at
> org.apache.activemq.broker.region.RegionBroker.addConsumer(RegionBroker.java:447)
> - locked < 0x00000007603866f8> (a
> org.apache.activemq.broker.region.RegionBroker$1)
> at
> org.apache.activemq.broker.jmx.ManagedRegionBroker.addConsumer(ManagedRegionBroker.java:240)
> at
> org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:89)
> at
> org.apache.activemq.advisory.AdvisoryBroker.addConsumer(AdvisoryBroker.java:91)
> at
> org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:89)
> at
> org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:89)
> at
> org.apache.activemq.broker.MutableBrokerFilter.addConsumer(MutableBrokerFilter.java:95)
> at
> org.apache.activemq.broker.TransportConnection.processAddConsumer(TransportConnection.java:550)
> at org.apache.activemq.command.ConsumerInfo.visit(ConsumerInfo.java:349)
> at
> org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:311)
> at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:185)
> at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:228)
> - locked < 0x0000000761279ad8> (a
> org.apache.activemq.transport.InactivityMonitor$1)
> at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
> at java.lang.Thread.run(Thread.java:662)
>
>"ActiveMQ Transport: tcp:///{IP}:45395" daemon prio=10 tid=0x000000005ded2000
>nid=0xe22 waiting on condition [0x00002b3c84a07000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x000000076037c188> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:811)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:941)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1261)
> at
> java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:594)
> at
> org.apache.activemq.store.jdbc.adapter.DefaultJDBCAdapter.doGetDurableSubscriberMessageCount(DefaultJDBCAdapter.java:575)
> at
> org.apache.activemq.store.jdbc.JDBCTopicMessageStore.getMessageCount(JDBCTopicMessageStore.java:223)
> at
> org.apache.activemq.store.ProxyTopicMessageStore.getMessageCount(ProxyTopicMessageStore.java:116)
> at
> org.apache.activemq.broker.region.cursors.TopicStorePrefetch.getStoreSize(TopicStorePrefetch.java:77)
> - locked < 0x00000007603869d0> (a
> org.apache.activemq.broker.region.cursors.TopicStorePrefetch)
> at
> org.apache.activemq.broker.region.cursors.AbstractStoreCursor.start(AbstractStoreCursor.java:59)
> - locked < 0x00000007603869d0> (a
> org.apache.activemq.broker.region.cursors.TopicStorePrefetch)
> at
> org.apache.activemq.broker.region.cursors.StoreDurableSubscriberCursor.start(StoreDurableSubscriberCursor.java:81)
> - locked < 0x00000007603868e8> (a
> org.apache.activemq.broker.region.cursors.StoreDurableSubscriberCursor)
> at
> org.apache.activemq.broker.region.DurableTopicSubscription.activate(DurableTopicSubscription.java:137)
> - locked < 0x00000007603868e8> (a
> org.apache.activemq.broker.region.cursors.StoreDurableSubscriberCursor)
> at
> org.apache.activemq.broker.region.TopicRegion.addConsumer(TopicRegion.java:108)
> at
> org.apache.activemq.broker.region.RegionBroker.addConsumer(RegionBroker.java:447)
> - locked < 0x00000007603866f8> (a
> org.apache.activemq.broker.region.RegionBroker$1)
> at
> org.apache.activemq.broker.jmx.ManagedRegionBroker.addConsumer(ManagedRegionBroker.java:240)
> at
> org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:89)
> at
> org.apache.activemq.advisory.AdvisoryBroker.addConsumer(AdvisoryBroker.java:91)
> at
> org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:89)
> at
> org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:89)
> at
> org.apache.activemq.broker.MutableBrokerFilter.addConsumer(MutableBrokerFilter.java:95)
> at
> org.apache.activemq.broker.TransportConnection.processAddConsumer(TransportConnection.java:550)
> at org.apache.activemq.command.ConsumerInfo.visit(ConsumerInfo.java:349)
> at
> org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:311)
> at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:185)
> at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:228)
> - locked < 0x0000000761279ad8> (a
> org.apache.activemq.transport.InactivityMonitor$1)
> at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
> at java.lang.Thread.run(Thread.java:662)
>
>
>Regards,
>P. Tlholoe
>073 342 4393
>
>
>To read FirstRand Bank's Disclaimer for this email click on the following
>address or copy into your Internet browser:
>https://www.fnb.co.za/disclaimer.html
>
>If you are unable to access the Disclaimer, send a blank e-mail to
>firstrandbankdisclai...@fnb.co.za and we will send you a copy of the
>Disclaimer.
