you are experiencing: https://issues.apache.org/jira/browse/AMQ-3434
that is resolved in the current 5.6-SNAPSHOT if you want to give that a whirl On 11 August 2011 03:47, kaustubh khasnis <[email protected]> wrote: > Hi, > I was running a performance test in which I tried to add a backlog of around > 4-5 M messages over 10 virtual topic,pushed by around 100 producers and then > tried to drain them by equal number of consumers , with 10 consumers forming > a group of virtual topic consumer. in middle of draining I encountered with > following exception in broker log. Can anyone please tell me what went > wrong?? After the exception as well, the broker and consumers continued > their normal operations. > > 2011-08-11 02:36:19,067 ERROR > [org.apache.activemq.broker.region.cursors.AbstractStoreCursor] > QueueStorePrefetch1976196437 - Failed to fill batch > java.lang.NullPointerException > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110) > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121) > at > org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491) > at org.apache.kahadb.page.Transaction.execute(Transaction.java:728) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485) > at > org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88) > at > org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145) > at > org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131) > at > org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713) > at > org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932) > at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440) > at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643) > at > org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394) > at > org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383) > at > org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87) > at > org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465) > at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179) > at > org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116) > at > org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69) > at > org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218) > at > org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127) > at > org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) > at java.lang.Thread.run(Thread.java:662) > 2011-08-11 02:36:19,070 ERROR > [org.apache.activemq.broker.region.cursors.AbstractStoreCursor] > QueueStorePrefetch1976196437 - Failed to fill batch > java.lang.RuntimeException: java.lang.NullPointerException > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145) > at > org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131) > at > org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713) > at > org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932) > at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440) > at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643) > at > org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394) > at > org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383) > at > org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87) > at > org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465) > at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179) > at > org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116) > at > org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69) > at > org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218) > at > org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127) > at > org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) > at java.lang.Thread.run(Thread.java:662) > Caused by: java.lang.NullPointerException > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110) > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121) > at > org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491) > at org.apache.kahadb.page.Transaction.execute(Transaction.java:728) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485) > at > org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88) > at > org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260) > ... 25 more > 2011-08-11 02:36:19,071 ERROR [org.apache.activemq.broker.region.Queue] > Failed to page in more queue messages > java.lang.RuntimeException: java.lang.RuntimeException: > java.lang.NullPointerException > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:148) > at > org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131) > at > org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713) > at > org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932) > at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440) > at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643) > at > org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394) > at > org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383) > at > org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87) > at > org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465) > at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179) > at > org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116) > at > org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69) > at > org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218) > at > org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127) > at > org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) > at java.lang.Thread.run(Thread.java:662) > Caused by: java.lang.RuntimeException: java.lang.NullPointerException > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145) > ... 24 more > Caused by: java.lang.NullPointerException > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110) > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121) > at > org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491) > at org.apache.kahadb.page.Transaction.execute(Transaction.java:728) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485) > at > org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88) > at > org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260) > ... 25 more > 2011-08-11 02:36:19,087 ERROR > [org.apache.activemq.broker.region.cursors.AbstractStoreCursor] > QueueStorePrefetch1976196437 - Failed to fill batch > java.lang.NullPointerException > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110) > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121) > at > org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491) > at org.apache.kahadb.page.Transaction.execute(Transaction.java:728) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485) > at > org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88) > at > org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145) > at > org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131) > at > org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713) > at > org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932) > at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440) > at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643) > at > org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394) > at > org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383) > at > org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87) > at > org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465) > at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179) > at > org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116) > at > org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69) > at > org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218) > at > org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127) > at > org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) > at java.lang.Thread.run(Thread.java:662) > 2011-08-11 02:36:19,089 ERROR > [org.apache.activemq.broker.region.cursors.AbstractStoreCursor] > QueueStorePrefetch1976196437 - Failed to fill batch > java.lang.RuntimeException: java.lang.NullPointerException > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145) > at > org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131) > at > org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713) > at > org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932) > at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440) > at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643) > at > org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394) > at > org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383) > at > org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87) > at > org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465) > at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179) > at > org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116) > at > org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69) > at > org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218) > at > org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127) > at > org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) > at java.lang.Thread.run(Thread.java:662) > Caused by: java.lang.NullPointerException > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110) > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121) > at > org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491) > at org.apache.kahadb.page.Transaction.execute(Transaction.java:728) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485) > at > org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88) > at > org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260) > ... 25 more > 2011-08-11 02:36:19,090 ERROR [org.apache.activemq.broker.region.Queue] > Failed to page in more queue messages > java.lang.RuntimeException: java.lang.RuntimeException: > java.lang.NullPointerException > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:148) > at > org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131) > at > org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713) > at > org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932) > at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440) > at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643) > at > org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394) > at > org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383) > at > org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197) > at > org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77) > at > org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87) > at > org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465) > at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229) > at > org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306) > at > org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179) > at > org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116) > at > org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69) > at > org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218) > at > org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127) > at > org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) > at java.lang.Thread.run(Thread.java:662) > Caused by: java.lang.RuntimeException: java.lang.NullPointerException > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145) > ... 24 more > Caused by: java.lang.NullPointerException > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110) > at > org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121) > at > org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491) > at org.apache.kahadb.page.Transaction.execute(Transaction.java:728) > at > org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485) > at > org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88) > at > org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97) > at > org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260) > ... 25 more > > > --Kaustubh > -- http://fusesource.com http://blog.garytully.com
