Hello, We have been running Jackrabbit 1.4.4 for some time with no problem.
But today I have an issue with many threads blocked. A thread dump showed that there is a deadlock in lucene. I created ticket http://issues.apache.org/jira/browse/JCR-2426 Any help is appreciated. This has happened in a production setup, which we had to restart. Thanks, Antonio ------------------------- Found one Java-level deadlock: ============================= "RMI TCP Connection(106261)-10.196.27.37": waiting to lock monitor 0x03764818 (object 0x80c376b0, a org.apache.lucene.store.FSDirectory$FSIndexInput), which is held by "jmssecondaryApplnJobExecutor-7" "jmssecondaryApplnJobExecutor-7": waiting to lock monitor 0x050ad618 (object 0x808fa710, a org.apache.lucene.index.SegmentReader), which is held by "IndexMerger" "IndexMerger": waiting to lock monitor 0x03764818 (object 0x80c376b0, a org.apache.lucene.store.FSDirectory$FSIndexInput), which is held by "jmssecondaryApplnJobExecutor-7" Java stack information for the threads listed above: =================================================== "RMI TCP Connection(106261)-10.196.27.37": at org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235) - waiting to lock <0x80c376b0> (a org.apache.lucene.store.FSDirectory$FSIndexInput) at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:143) at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38) at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:57) at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63) at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:122) at org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:153) at org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:164) at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:158) at org.apache.lucene.index.SegmentReader.docFreq(SegmentReader.java:352) at org.apache.lucene.index.FilterIndexReader.docFreq(FilterIndexReader.java:176) at org.apache.lucene.index.FilterIndexReader.docFreq(FilterIndexReader.java:176) at org.apache.lucene.index.FilterIndexReader.docFreq(FilterIndexReader.java:176) at org.apache.lucene.index.MultiReader.docFreq(MultiReader.java:220) at org.apache.lucene.index.MultiReader.docFreq(MultiReader.java:220) at org.apache.lucene.search.IndexSearcher.docFreq(IndexSearcher.java:87) at org.apache.lucene.search.Similarity.idf(Similarity.java:459) at org.apache.lucene.search.TermQuery$TermWeight.<init>(TermQuery.java:44) at org.apache.lucene.search.TermQuery.createWeight(TermQuery.java:145) at org.apache.lucene.search.BooleanQuery$BooleanWeight.<init>(BooleanQuery.java:187) at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:353) at org.apache.lucene.search.BooleanQuery$BooleanWeight.<init>(BooleanQuery.java:187) at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:353) at org.apache.lucene.search.Query.weight(Query.java:95) at org.apache.lucene.search.Hits.<init>(Hits.java:50) at org.apache.lucene.search.Searcher.search(Searcher.java:46) at org.apache.lucene.search.Searcher.search(Searcher.java:38) at org.apache.jackrabbit.core.query.lucene.SearchIndex.executeQuery(SearchIndex.java:691) at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.executeQuery(QueryResultImpl.java:242) at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.getResults(QueryResultImpl.java:290) at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.<init>(QueryResultImpl.java:192) at org.apache.jackrabbit.core.query.lucene.QueryImpl.execute(QueryImpl.java:138) at org.apache.jackrabbit.core.query.QueryImpl.execute(QueryImpl.java:176) at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl$2.execute(InventoryDiscoveryServiceImpl.java:422) at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.execute(InventoryDiscoveryServiceImpl.java:316) at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.queryData(InventoryDiscoveryServiceImpl.java:407) at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.queryData(InventoryDiscoveryServiceImpl.java:366) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:36) at sun.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:243) at javax.management.modelmbean.RequiredModelMBean.invokeMethod(RequiredModelMBean.java:1074) at javax.management.modelmbean.RequiredModelMBean.invoke(RequiredModelMBean.java:955) at org.springframework.jmx.export.SpringModelMBean.invoke(SpringModelMBean.java:88) at org.jboss.mx.server.RawDynamicInvoker.invoke(RawDynamicInvoker.java:164) at org.jboss.mx.modelmbean.RequiredModelMBeanInvoker.invoke(RequiredModelMBeanInvoker.java:127) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) at sun.reflect.GeneratedMethodAccessor6569.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.jmx.connector.invoker.InvokerAdaptorService.invoke(InvokerAdaptorService.java:266) at sun.reflect.GeneratedMethodAccessor163.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94) at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:133) at org.jboss.mx.server.Invocation.invoke(Invocation.java:88) at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:142) at com.alcatel.axs.applications.nelist.SnmpStatsInterceptor.invoke(SnmpStatsInterceptor.java:21) at com.alcatel.axs.container.jmx.PayloadServerInterceptor.invoke(PayloadServerInterceptor.java:32) at com.alcatel.axs.app.security.session.SessionIdPostAuthenticationServerInterceptor.invoke(SessionIdPostAuthenticationServerInterceptor.java:118) at org.jboss.jmx.connector.invoker.AuthenticationInterceptor.invoke(AuthenticationInterceptor.java:108) at com.alcatel.axs.app.security.session.AxsAuthenticationInterceptor.invoke(AxsAuthenticationInterceptor.java:80) at com.alcatel.axs.app.security.session.SessionIdPreAuthenticationServerInterceptor.invoke(SessionIdPreAuthenticationServerInterceptor.java:86) at org.jboss.mx.server.Invocation.invoke(Invocation.java:88) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) at org.jboss.invocation.jrmp.server.JRMPInvokerHA.invoke(JRMPInvokerHA.java:177) at sun.reflect.GeneratedMethodAccessor636.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305) at sun.rmi.transport.Transport$1.run(Transport.java:159) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Transport.java:155) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "jmssecondaryApplnJobExecutor-7": at org.apache.lucene.index.SegmentReader.hasNorms(SegmentReader.java:421) - waiting to lock <0x808fa710> (a org.apache.lucene.index.SegmentReader) at org.apache.lucene.index.FilterIndexReader.hasNorms(FilterIndexReader.java:147) at org.apache.lucene.index.FilterIndexReader.hasNorms(FilterIndexReader.java:147) at org.apache.lucene.index.FilterIndexReader.hasNorms(FilterIndexReader.java:147) at org.apache.lucene.index.MultiReader.hasNorms(MultiReader.java:161) at org.apache.lucene.index.MultiReader.hasNorms(MultiReader.java:161) at org.apache.lucene.index.MultiReader.norms(MultiReader.java:177) - locked <0x80f07f60> (a org.apache.jackrabbit.core.query.lucene.SearchIndex$CombinedIndexReader) at org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:69) at org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:233) at org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:233) at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143) at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:113) at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:74) at org.apache.lucene.search.Hits.<init>(Hits.java:53) at org.apache.lucene.search.Searcher.search(Searcher.java:46) at org.apache.lucene.search.Searcher.search(Searcher.java:38) at org.apache.jackrabbit.core.query.lucene.SearchIndex.executeQuery(SearchIndex.java:691) at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.executeQuery(QueryResultImpl.java:242) at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.getResults(QueryResultImpl.java:290) at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.<init>(QueryResultImpl.java:192) at org.apache.jackrabbit.core.query.lucene.QueryImpl.execute(QueryImpl.java:138) at org.apache.jackrabbit.core.query.QueryImpl.execute(QueryImpl.java:176) at com.alcatel.axs.app.idm.nelist.InventoryNeListServiceImpl.findNeNode(InventoryNeListServiceImpl.java:621) at com.alcatel.axs.app.idm.nelist.InventoryNeListServiceImpl.collectionBegin(InventoryNeListServiceImpl.java:137) at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.doDiscoveryNow(InventoryDiscoveryServiceImpl.java:219) at sun.reflect.GeneratedMethodAccessor7068.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:36) at sun.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:243) at javax.management.modelmbean.RequiredModelMBean.invokeMethod(RequiredModelMBean.java:1074) at javax.management.modelmbean.RequiredModelMBean.invoke(RequiredModelMBean.java:955) at org.springframework.jmx.export.SpringModelMBean.invoke(SpringModelMBean.java:88) at org.jboss.mx.server.RawDynamicInvoker.invoke(RawDynamicInvoker.java:164) at org.jboss.mx.modelmbean.RequiredModelMBeanInvoker.invoke(RequiredModelMBeanInvoker.java:127) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) at org.jboss.system.server.jmx.LazyMBeanServer.invoke(LazyMBeanServer.java:291) at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:288) at $Proxy638.doDiscoveryNow(Unknown Source) at com.alcatel.axs.app.idm.InventoryJob.execute(InventoryJob.java:41) at com.alcatel.axs.container.jobmanager.ManagedJob.run(ManagedJob.java:55) at com.alcatel.axs.container.jobmanager.JMSJobExecutor.onMessage(JMSJobExecutor.java:50) at org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:531) at org.springframework.jms.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:466) at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:435) at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:322) at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:260) at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:944) at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:868) at java.lang.Thread.run(Thread.java:619) "IndexMerger": at org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235) - waiting to lock <0x80c376b0> (a org.apache.lucene.store.FSDirectory$FSIndexInput) at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:143) at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38) at org.apache.lucene.store.IndexInput.readInt(IndexInput.java:47) at org.apache.lucene.store.IndexInput.readLong(IndexInput.java:70) at org.apache.lucene.index.FieldsReader.doc(FieldsReader.java:104) at org.apache.lucene.index.SegmentReader.document(SegmentReader.java:333) - locked <0x808fa710> (a org.apache.lucene.index.SegmentReader) at org.apache.lucene.index.FilterIndexReader.document(FilterIndexReader.java:130) at org.apache.lucene.index.FilterIndexReader.document(FilterIndexReader.java:130) at org.apache.lucene.index.FilterIndexReader.document(FilterIndexReader.java:130) at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:207) at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:97) at org.apache.lucene.index.IndexWriter.addIndexes(IndexWriter.java:1632) - locked <0x808f96d0> (a org.apache.lucene.index.IndexWriter) at org.apache.jackrabbit.core.query.lucene.PersistentIndex.addIndexes(PersistentIndex.java:107) at org.apache.jackrabbit.core.query.lucene.IndexMerger.run(IndexMerger.java:298) Found 1 deadlock.
