This is mildly unfortunately. Probably should migrate to LevelDB… :) But this is definitely a bug.
Essentially what happened was that I tried to purge a queue with a large number of message. ActiveMQ went to 100% CPU and stopped processing messages or making any progress. I stopped ActiveMQ and then restarted it… and then it deadlocked. Found one Java-level deadlock: ============================= "JobScheduler:JMS": waiting for ownable synchronizer 0x00000000f6c60fa8, (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync), which is held by "main" "main": waiting for ownable synchronizer 0x00000000c0027260, (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync), which is held by "JobScheduler:JMS" Java stack information for the threads listed above: =================================================== "JobScheduler:JMS": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f6c60fa8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:964) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:731) at org.apache.activemq.broker.region.AbstractRegion.getDestinations(AbstractRegion.java:243) at org.apache.activemq.broker.region.RegionBroker.getDestinations(RegionBroker.java:140) at org.apache.activemq.store.kahadb.KahaDBStore.rollbackStatsOnDuplicate(KahaDBStore.java:284) at org.apache.activemq.store.kahadb.MessageDatabase.updateIndex(MessageDatabase.java:1316) at org.apache.activemq.store.kahadb.MessageDatabase$11.execute(MessageDatabase.java:1140) at org.apache.activemq.store.kahadb.disk.page.Transaction.execute(Transaction.java:779) at org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:1137) at org.apache.activemq.store.kahadb.MessageDatabase$10.visit(MessageDatabase.java:1074) at org.apache.activemq.store.kahadb.data.KahaAddMessageCommand.visit(KahaAddMessageCommand.java:241) at org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:1071) at org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:978) at org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:958) at org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.addMessage(KahaDBStore.java:424) at org.apache.activemq.store.kahadb.KahaDBTransactionStore.addMessage(KahaDBTransactionStore.java:388) at org.apache.activemq.store.kahadb.KahaDBTransactionStore$1.addMessage(KahaDBTransactionStore.java:160) at org.apache.activemq.broker.region.Queue.doMessageSend(Queue.java:913) at org.apache.activemq.broker.region.Queue.send(Queue.java:733) at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:424) at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:445) at org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:297) at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:147) at org.apache.activemq.broker.scheduler.SchedulerBroker.scheduledJob(SchedulerBroker.java:282) at org.apache.activemq.store.kahadb.scheduler.JobSchedulerImpl.fireJob(JobSchedulerImpl.java:485) at org.apache.activemq.store.kahadb.scheduler.JobSchedulerImpl.mainLoop(JobSchedulerImpl.java:535) at org.apache.activemq.store.kahadb.scheduler.JobSchedulerImpl.run(JobSchedulerImpl.java:492) at java.lang.Thread.run(Thread.java:724) "main": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000c0027260> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:945) at org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.resetBatching(KahaDBStore.java:618) at org.apache.activemq.store.ProxyMessageStore.resetBatching(ProxyMessageStore.java:111) at org.apache.activemq.broker.region.cursors.QueueStorePrefetch.resetBatch(QueueStorePrefetch.java:92) at org.apache.activemq.broker.region.cursors.AbstractStoreCursor.start(AbstractStoreCursor.java:61) - locked <0x00000000f98a12a8> (a org.apache.activemq.broker.region.cursors.QueueStorePrefetch) at org.apache.activemq.broker.region.cursors.StoreQueueCursor.start(StoreQueueCursor.java:72) - locked <0x00000000f98a1258> (a org.apache.activemq.broker.region.cursors.StoreQueueCursor) at org.apache.activemq.broker.region.Queue.start(Queue.java:1037) at org.apache.activemq.broker.region.AbstractRegion.addDestination(AbstractRegion.java:141) at org.apache.activemq.broker.region.RegionBroker.addDestination(RegionBroker.java:325) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167) at org.apache.activemq.advisory.AdvisoryBroker.addDestination(AdvisoryBroker.java:184) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167) at org.apache.activemq.broker.MutableBrokerFilter.addDestination(MutableBrokerFilter.java:172) at org.apache.activemq.broker.region.AbstractRegion.start(AbstractRegion.java:99) at org.apache.activemq.broker.region.RegionBroker.start(RegionBroker.java:190) at org.apache.activemq.broker.jmx.ManagedRegionBroker.start(ManagedRegionBroker.java:121) at org.apache.activemq.broker.BrokerFilter.start(BrokerFilter.java:182) at org.apache.activemq.broker.scheduler.SchedulerBroker.start(SchedulerBroker.java:81) at org.apache.activemq.broker.BrokerFilter.start(BrokerFilter.java:182) at org.apache.activemq.broker.BrokerFilter.start(BrokerFilter.java:182) at org.apache.activemq.broker.TransactionBroker.start(TransactionBroker.java:120) at org.apache.activemq.broker.BrokerService$5.start(BrokerService.java:2177) at org.apache.activemq.broker.BrokerService.doStartBroker(BrokerService.java:675) at org.apache.activemq.broker.BrokerService.startBroker(BrokerService.java:659) at org.apache.activemq.broker.BrokerService.start(BrokerService.java:595) at org.apache.activemq.xbean.XBeanBrokerService.afterPropertiesSet(XBeanBrokerService.java:73) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeCustomInitMethod(AbstractAutowireCapableBeanFactory.java:1638) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1579) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1509) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:521) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:458) at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:296) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223) - locked <0x00000000c0051148> (a java.util.concurrent.ConcurrentHashMap) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:293) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194) at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:628) at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932) at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479) - locked <0x00000000c0051240> (a java.lang.Object) at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:64) at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:52) at org.apache.activemq.xbean.XBeanBrokerFactory$1.<init>(XBeanBrokerFactory.java:104) at org.apache.activemq.xbean.XBeanBrokerFactory.createApplicationContext(XBeanBrokerFactory.java:104) at org.apache.activemq.xbean.XBeanBrokerFactory.createBroker(XBeanBrokerFactory.java:67) at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:71) at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:54) at org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:87) at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:57) at org.apache.activemq.console.command.ShellCommand.runTask(ShellCommand.java:150) at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:57) at org.apache.activemq.console.command.ShellCommand.main(ShellCommand.java:104) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.activemq.console.Main.runTaskClass(Main.java:262) at org.apache.activemq.console.Main.main(Main.java:115) Found 1 deadlock. -- Founder/CEO Spinn3r.com Location: *San Francisco, CA* blog: http://burtonator.wordpress.com … or check out my Google+ profile <https://plus.google.com/102718274791889610666/posts> <http://spinn3r.com>
